1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
8
9define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1,
10; GFX7-LABEL: udot8_acc32:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
13; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
14; GFX7-NEXT:    s_mov_b32 s7, 0xf000
15; GFX7-NEXT:    s_mov_b32 s6, -1
16; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
17; GFX7-NEXT:    s_load_dword s10, s[10:11], 0x0
18; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
19; GFX7-NEXT:    s_load_dword s21, s[4:5], 0x0
20; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX7-NEXT:    s_lshr_b32 s11, s10, 28
22; GFX7-NEXT:    s_bfe_u32 s15, s10, 0x40018
23; GFX7-NEXT:    s_bfe_u32 s16, s10, 0x40014
24; GFX7-NEXT:    s_bfe_u32 s17, s10, 0x40010
25; GFX7-NEXT:    s_bfe_u32 s18, s10, 0x4000c
26; GFX7-NEXT:    s_bfe_u32 s19, s10, 0x40008
27; GFX7-NEXT:    s_bfe_u32 s20, s10, 0x40004
28; GFX7-NEXT:    s_and_b32 s10, s10, 15
29; GFX7-NEXT:    s_lshr_b32 s1, s0, 28
30; GFX7-NEXT:    s_bfe_u32 s2, s0, 0x40018
31; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40014
32; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40010
33; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x4000c
34; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40008
35; GFX7-NEXT:    s_bfe_u32 s14, s0, 0x40004
36; GFX7-NEXT:    s_and_b32 s0, s0, 15
37; GFX7-NEXT:    v_mov_b32_e32 v0, s10
38; GFX7-NEXT:    v_mov_b32_e32 v1, s21
39; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v0, v1
40; GFX7-NEXT:    v_mov_b32_e32 v1, s20
41; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v1, v0
42; GFX7-NEXT:    v_mov_b32_e32 v1, s19
43; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v1, v0
44; GFX7-NEXT:    v_mov_b32_e32 v1, s18
45; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
46; GFX7-NEXT:    v_mov_b32_e32 v1, s17
47; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
48; GFX7-NEXT:    v_mov_b32_e32 v1, s16
49; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
50; GFX7-NEXT:    v_mov_b32_e32 v1, s15
51; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
52; GFX7-NEXT:    v_mov_b32_e32 v1, s11
53; GFX7-NEXT:    v_mad_u32_u24 v0, s1, v1, v0
54; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
55; GFX7-NEXT:    s_endpgm
56;
57; GFX8-LABEL: udot8_acc32:
58; GFX8:       ; %bb.0: ; %entry
59; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
60; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
61; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
62; GFX8-NEXT:    s_load_dword s6, s[6:7], 0x0
63; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
64; GFX8-NEXT:    s_load_dword s19, s[0:1], 0x0
65; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
66; GFX8-NEXT:    s_lshr_b32 s7, s6, 28
67; GFX8-NEXT:    s_bfe_u32 s13, s6, 0x40018
68; GFX8-NEXT:    s_bfe_u32 s14, s6, 0x40014
69; GFX8-NEXT:    s_bfe_u32 s15, s6, 0x40010
70; GFX8-NEXT:    s_bfe_u32 s16, s6, 0x4000c
71; GFX8-NEXT:    s_bfe_u32 s17, s6, 0x40008
72; GFX8-NEXT:    s_bfe_u32 s18, s6, 0x40004
73; GFX8-NEXT:    s_and_b32 s6, s6, 15
74; GFX8-NEXT:    s_lshr_b32 s4, s2, 28
75; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x40018
76; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40014
77; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40010
78; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x4000c
79; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40008
80; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40004
81; GFX8-NEXT:    s_and_b32 s2, s2, 15
82; GFX8-NEXT:    v_mov_b32_e32 v0, s6
83; GFX8-NEXT:    v_mov_b32_e32 v1, s19
84; GFX8-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
85; GFX8-NEXT:    v_mov_b32_e32 v1, s18
86; GFX8-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
87; GFX8-NEXT:    v_mov_b32_e32 v1, s17
88; GFX8-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
89; GFX8-NEXT:    v_mov_b32_e32 v1, s16
90; GFX8-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
91; GFX8-NEXT:    v_mov_b32_e32 v1, s15
92; GFX8-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
93; GFX8-NEXT:    v_mov_b32_e32 v1, s14
94; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
95; GFX8-NEXT:    v_mov_b32_e32 v1, s13
96; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
97; GFX8-NEXT:    v_mov_b32_e32 v1, s7
98; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
99; GFX8-NEXT:    v_mov_b32_e32 v0, s0
100; GFX8-NEXT:    v_mov_b32_e32 v1, s1
101; GFX8-NEXT:    flat_store_dword v[0:1], v2
102; GFX8-NEXT:    s_endpgm
103;
104; GFX9-LABEL: udot8_acc32:
105; GFX9:       ; %bb.0: ; %entry
106; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
107; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
108; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
109; GFX9-NEXT:    s_load_dword s6, s[6:7], 0x0
110; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
111; GFX9-NEXT:    s_load_dword s19, s[0:1], 0x0
112; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
113; GFX9-NEXT:    s_lshr_b32 s7, s6, 28
114; GFX9-NEXT:    s_bfe_u32 s13, s6, 0x40018
115; GFX9-NEXT:    s_bfe_u32 s14, s6, 0x40014
116; GFX9-NEXT:    s_bfe_u32 s15, s6, 0x40010
117; GFX9-NEXT:    s_bfe_u32 s16, s6, 0x4000c
118; GFX9-NEXT:    s_bfe_u32 s17, s6, 0x40008
119; GFX9-NEXT:    s_bfe_u32 s18, s6, 0x40004
120; GFX9-NEXT:    s_and_b32 s6, s6, 15
121; GFX9-NEXT:    s_lshr_b32 s4, s2, 28
122; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40018
123; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
124; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40010
125; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x4000c
126; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40008
127; GFX9-NEXT:    s_bfe_u32 s12, s2, 0x40004
128; GFX9-NEXT:    s_and_b32 s2, s2, 15
129; GFX9-NEXT:    v_mov_b32_e32 v0, s6
130; GFX9-NEXT:    v_mov_b32_e32 v1, s19
131; GFX9-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
132; GFX9-NEXT:    v_mov_b32_e32 v1, s18
133; GFX9-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
134; GFX9-NEXT:    v_mov_b32_e32 v1, s17
135; GFX9-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
136; GFX9-NEXT:    v_mov_b32_e32 v1, s16
137; GFX9-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
138; GFX9-NEXT:    v_mov_b32_e32 v1, s15
139; GFX9-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
140; GFX9-NEXT:    v_mov_b32_e32 v1, s14
141; GFX9-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
142; GFX9-NEXT:    v_mov_b32_e32 v1, s13
143; GFX9-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
144; GFX9-NEXT:    v_mov_b32_e32 v1, s7
145; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
146; GFX9-NEXT:    v_mov_b32_e32 v0, s0
147; GFX9-NEXT:    v_mov_b32_e32 v1, s1
148; GFX9-NEXT:    global_store_dword v[0:1], v2, off
149; GFX9-NEXT:    s_endpgm
150;
151; GFX9-DL-LABEL: udot8_acc32:
152; GFX9-DL:       ; %bb.0: ; %entry
153; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
154; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
155; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
156; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
157; GFX9-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
158; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
159; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
160; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
161; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
162; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
163; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s6
164; GFX9-DL-NEXT:    v_dot8_u32_u4 v2, s4, v2, v3
165; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
166; GFX9-DL-NEXT:    s_endpgm
167;
168; GFX10-DL-LABEL: udot8_acc32:
169; GFX10-DL:       ; %bb.0: ; %entry
170; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
171; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
172; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
173; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
174; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
175; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
176; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
177; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
178; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
179; GFX10-DL-NEXT:    v_dot8_u32_u4 v2, s4, s5, v0
180; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
181; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
182; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
183; GFX10-DL-NEXT:    s_endpgm
184                                       <8 x i4> addrspace(1)* %src2,
185                                       i32 addrspace(1)* nocapture %dst) {
186entry:
187  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
188  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
189
190  %v1e0 = extractelement <8 x i4> %vec1, i64 0
191  %cv1e0 = zext i4 %v1e0 to i32
192  %v2e0 = extractelement <8 x i4> %vec2, i64 0
193  %cv2e0 = zext i4 %v2e0 to i32
194  %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
195
196  %v1e1 = extractelement <8 x i4> %vec1, i64 1
197  %cv1e1 = zext i4 %v1e1 to i32
198  %v2e1 = extractelement <8 x i4> %vec2, i64 1
199  %cv2e1 = zext i4 %v2e1 to i32
200  %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
201
202  %v1e2 = extractelement <8 x i4> %vec1, i64 2
203  %cv1e2 = zext i4 %v1e2 to i32
204  %v2e2 = extractelement <8 x i4> %vec2, i64 2
205  %cv2e2 = zext i4 %v2e2 to i32
206  %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
207
208  %v1e3 = extractelement <8 x i4> %vec1, i64 3
209  %cv1e3 = zext i4 %v1e3 to i32
210  %v2e3 = extractelement <8 x i4> %vec2, i64 3
211  %cv2e3 = zext i4 %v2e3 to i32
212  %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
213
214  %v1e4 = extractelement <8 x i4> %vec1, i64 4
215  %cv1e4 = zext i4 %v1e4 to i32
216  %v2e4 = extractelement <8 x i4> %vec2, i64 4
217  %cv2e4 = zext i4 %v2e4 to i32
218  %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
219
220  %v1e5 = extractelement <8 x i4> %vec1, i64 5
221  %cv1e5 = zext i4 %v1e5 to i32
222  %v2e5 = extractelement <8 x i4> %vec2, i64 5
223  %cv2e5 = zext i4 %v2e5 to i32
224  %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
225
226  %v1e6 = extractelement <8 x i4> %vec1, i64 6
227  %cv1e6 = zext i4 %v1e6 to i32
228  %v2e6 = extractelement <8 x i4> %vec2, i64 6
229  %cv2e6 = zext i4 %v2e6 to i32
230  %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
231
232  %v1e7 = extractelement <8 x i4> %vec1, i64 7
233  %cv1e7 = zext i4 %v1e7 to i32
234  %v2e7 = extractelement <8 x i4> %vec2, i64 7
235  %cv2e7 = zext i4 %v2e7 to i32
236  %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
237
238  %acc = load i32, i32 addrspace(1)* %dst, align 4
239  %add1 = add i32 %mul0, %acc
240  %add2 = add i32 %add1, %mul1
241  %add3 = add i32 %add2, %mul2
242  %add4 = add i32 %add3, %mul3
243  %add5 = add i32 %add4, %mul4
244  %add6 = add i32 %add5, %mul5
245  %add7 = add i32 %add6, %mul6
246  %add8 = add i32 %add7, %mul7
247
248  store i32 %add8, i32 addrspace(1)* %dst, align 4
249  ret void
250}
251
252; TODO: Remove the unnecessary instruction(that is zero-extending the
253; 2nd MAD) to have the pattern-recognizer to kick in.
254define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
255; GFX7-LABEL: udot8_acc16:
256; GFX7:       ; %bb.0: ; %entry
257; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
258; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
259; GFX7-NEXT:    s_mov_b32 s7, 0xf000
260; GFX7-NEXT:    s_mov_b32 s6, -1
261; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
262; GFX7-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
263; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
264; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
265; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
266; GFX7-NEXT:    s_lshr_b32 s2, s0, 28
267; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40018
268; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40014
269; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40010
270; GFX7-NEXT:    s_bfe_u32 s18, s1, 0x4000c
271; GFX7-NEXT:    s_bfe_u32 s19, s1, 0x40008
272; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40004
273; GFX7-NEXT:    s_lshr_b32 s14, s1, 28
274; GFX7-NEXT:    s_and_b32 s1, s1, 15
275; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40018
276; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40014
277; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40010
278; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x4000c
279; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x40008
280; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40004
281; GFX7-NEXT:    s_and_b32 s0, s0, 15
282; GFX7-NEXT:    v_mov_b32_e32 v1, s1
283; GFX7-NEXT:    v_mov_b32_e32 v2, s20
284; GFX7-NEXT:    v_mov_b32_e32 v3, s19
285; GFX7-NEXT:    v_mov_b32_e32 v4, s18
286; GFX7-NEXT:    v_mov_b32_e32 v5, s17
287; GFX7-NEXT:    v_mov_b32_e32 v6, s16
288; GFX7-NEXT:    v_mov_b32_e32 v7, s15
289; GFX7-NEXT:    s_waitcnt vmcnt(0)
290; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
291; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v2, v0
292; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
293; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v4, v0
294; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v5, v0
295; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v6, v0
296; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v7, v0
297; GFX7-NEXT:    v_mov_b32_e32 v1, s14
298; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
299; GFX7-NEXT:    buffer_store_short v0, off, s[4:7], 0
300; GFX7-NEXT:    s_endpgm
301;
302; GFX8-LABEL: udot8_acc16:
303; GFX8:       ; %bb.0: ; %entry
304; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
305; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
306; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
307; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
308; GFX8-NEXT:    v_mov_b32_e32 v0, s0
309; GFX8-NEXT:    v_mov_b32_e32 v1, s1
310; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
311; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
312; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
313; GFX8-NEXT:    s_and_b32 s1, s2, 15
314; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40004
315; GFX8-NEXT:    v_mov_b32_e32 v3, s1
316; GFX8-NEXT:    v_mov_b32_e32 v4, s4
317; GFX8-NEXT:    s_and_b32 s1, s0, 15
318; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x40004
319; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40008
320; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x4000c
321; GFX8-NEXT:    v_mov_b32_e32 v5, s4
322; GFX8-NEXT:    s_bfe_u32 s6, s0, 0x40008
323; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40010
324; GFX8-NEXT:    v_mov_b32_e32 v6, s7
325; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x4000c
326; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40014
327; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x40010
328; GFX8-NEXT:    v_mov_b32_e32 v7, s8
329; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40018
330; GFX8-NEXT:    s_bfe_u32 s10, s0, 0x40014
331; GFX8-NEXT:    v_mov_b32_e32 v8, s9
332; GFX8-NEXT:    s_bfe_u32 s12, s0, 0x40018
333; GFX8-NEXT:    s_lshr_b32 s2, s2, 28
334; GFX8-NEXT:    v_mov_b32_e32 v9, s11
335; GFX8-NEXT:    s_lshr_b32 s0, s0, 28
336; GFX8-NEXT:    s_waitcnt vmcnt(0)
337; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
338; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
339; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff, v2
340; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
341; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
342; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
343; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
344; GFX8-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
345; GFX8-NEXT:    v_mov_b32_e32 v3, s2
346; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
347; GFX8-NEXT:    flat_store_short v[0:1], v2
348; GFX8-NEXT:    s_endpgm
349;
350; GFX9-LABEL: udot8_acc16:
351; GFX9:       ; %bb.0: ; %entry
352; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
353; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
354; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
355; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
356; GFX9-NEXT:    v_mov_b32_e32 v0, s0
357; GFX9-NEXT:    v_mov_b32_e32 v1, s1
358; GFX9-NEXT:    global_load_ushort v2, v[0:1], off
359; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
360; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
361; GFX9-NEXT:    s_and_b32 s1, s2, 15
362; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40004
363; GFX9-NEXT:    v_mov_b32_e32 v3, s1
364; GFX9-NEXT:    v_mov_b32_e32 v4, s4
365; GFX9-NEXT:    s_and_b32 s1, s0, 15
366; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40004
367; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40008
368; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x4000c
369; GFX9-NEXT:    v_mov_b32_e32 v5, s4
370; GFX9-NEXT:    s_bfe_u32 s6, s0, 0x40008
371; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40010
372; GFX9-NEXT:    v_mov_b32_e32 v6, s7
373; GFX9-NEXT:    s_bfe_u32 s4, s0, 0x4000c
374; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40014
375; GFX9-NEXT:    s_bfe_u32 s7, s0, 0x40010
376; GFX9-NEXT:    v_mov_b32_e32 v7, s8
377; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40018
378; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40014
379; GFX9-NEXT:    v_mov_b32_e32 v8, s9
380; GFX9-NEXT:    s_bfe_u32 s12, s0, 0x40018
381; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
382; GFX9-NEXT:    v_mov_b32_e32 v9, s11
383; GFX9-NEXT:    s_lshr_b32 s0, s0, 28
384; GFX9-NEXT:    s_waitcnt vmcnt(0)
385; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
386; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
387; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v2
388; GFX9-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
389; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
390; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
391; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
392; GFX9-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
393; GFX9-NEXT:    v_mov_b32_e32 v3, s2
394; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
395; GFX9-NEXT:    global_store_short v[0:1], v2, off
396; GFX9-NEXT:    s_endpgm
397;
398; GFX9-DL-LABEL: udot8_acc16:
399; GFX9-DL:       ; %bb.0: ; %entry
400; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
401; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
402; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
403; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
404; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
405; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
406; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
407; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
408; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
409; GFX9-DL-NEXT:    s_and_b32 s1, s2, 15
410; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40004
411; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
412; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
413; GFX9-DL-NEXT:    s_and_b32 s1, s0, 15
414; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
415; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40008
416; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x4000c
417; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s4
418; GFX9-DL-NEXT:    s_bfe_u32 s6, s0, 0x40008
419; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40010
420; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s7
421; GFX9-DL-NEXT:    s_bfe_u32 s4, s0, 0x4000c
422; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40014
423; GFX9-DL-NEXT:    s_bfe_u32 s7, s0, 0x40010
424; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s8
425; GFX9-DL-NEXT:    s_bfe_u32 s11, s2, 0x40018
426; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40014
427; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s9
428; GFX9-DL-NEXT:    s_bfe_u32 s12, s0, 0x40018
429; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 28
430; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s11
431; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 28
432; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
433; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
434; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
435; GFX9-DL-NEXT:    v_and_b32_e32 v2, 0xffff, v2
436; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
437; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
438; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
439; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
440; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
441; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
442; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
443; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
444; GFX9-DL-NEXT:    s_endpgm
445;
446; GFX10-DL-LABEL: udot8_acc16:
447; GFX10-DL:       ; %bb.0: ; %entry
448; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
449; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
450; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
451; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
452; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
453; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
454; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
455; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
456; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
457; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
458; GFX10-DL-NEXT:    s_and_b32 s2, s0, 15
459; GFX10-DL-NEXT:    s_and_b32 s4, s1, 15
460; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
461; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40004
462; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
463; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
464; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40008
465; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40008
466; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
467; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x4000c
468; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x4000c
469; GFX10-DL-NEXT:    v_and_b32_e32 v2, 0xffff, v2
470; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
471; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40010
472; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40010
473; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
474; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
475; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40014
476; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
477; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
478; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40018
479; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
480; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
481; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
482; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
483; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
484; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
485; GFX10-DL-NEXT:    s_endpgm
486                                       <8 x i4> addrspace(1)* %src2,
487                                       i16 addrspace(1)* nocapture %dst) {
488entry:
489  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
490  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
491
492  %v1e0 = extractelement <8 x i4> %vec1, i64 0
493  %cv1e0 = zext i4 %v1e0 to i16
494  %v2e0 = extractelement <8 x i4> %vec2, i64 0
495  %cv2e0 = zext i4 %v2e0 to i16
496  %mul0 = mul nuw nsw i16 %cv1e0, %cv2e0
497
498  %v1e1 = extractelement <8 x i4> %vec1, i64 1
499  %cv1e1 = zext i4 %v1e1 to i16
500  %v2e1 = extractelement <8 x i4> %vec2, i64 1
501  %cv2e1 = zext i4 %v2e1 to i16
502  %mul1 = mul nuw nsw i16 %cv1e1, %cv2e1
503
504  %v1e2 = extractelement <8 x i4> %vec1, i64 2
505  %cv1e2 = zext i4 %v1e2 to i16
506  %v2e2 = extractelement <8 x i4> %vec2, i64 2
507  %cv2e2 = zext i4 %v2e2 to i16
508  %mul2 = mul nuw nsw i16 %cv1e2, %cv2e2
509
510  %v1e3 = extractelement <8 x i4> %vec1, i64 3
511  %cv1e3 = zext i4 %v1e3 to i16
512  %v2e3 = extractelement <8 x i4> %vec2, i64 3
513  %cv2e3 = zext i4 %v2e3 to i16
514  %mul3 = mul nuw nsw i16 %cv1e3, %cv2e3
515
516  %v1e4 = extractelement <8 x i4> %vec1, i64 4
517  %cv1e4 = zext i4 %v1e4 to i16
518  %v2e4 = extractelement <8 x i4> %vec2, i64 4
519  %cv2e4 = zext i4 %v2e4 to i16
520  %mul4 = mul nuw nsw i16 %cv1e4, %cv2e4
521
522  %v1e5 = extractelement <8 x i4> %vec1, i64 5
523  %cv1e5 = zext i4 %v1e5 to i16
524  %v2e5 = extractelement <8 x i4> %vec2, i64 5
525  %cv2e5 = zext i4 %v2e5 to i16
526  %mul5 = mul nuw nsw i16 %cv1e5, %cv2e5
527
528  %v1e6 = extractelement <8 x i4> %vec1, i64 6
529  %cv1e6 = zext i4 %v1e6 to i16
530  %v2e6 = extractelement <8 x i4> %vec2, i64 6
531  %cv2e6 = zext i4 %v2e6 to i16
532  %mul6 = mul nuw nsw i16 %cv1e6, %cv2e6
533
534  %v1e7 = extractelement <8 x i4> %vec1, i64 7
535  %cv1e7 = zext i4 %v1e7 to i16
536  %v2e7 = extractelement <8 x i4> %vec2, i64 7
537  %cv2e7 = zext i4 %v2e7 to i16
538  %mul7 = mul nuw nsw i16 %cv1e7, %cv2e7
539
540  %acc = load i16, i16 addrspace(1)* %dst, align 4
541  %add1 = add i16 %mul0, %acc
542  %add2 = add i16 %add1, %mul1
543  %add3 = add i16 %add2, %mul2
544  %add4 = add i16 %add3, %mul3
545  %add5 = add i16 %add4, %mul4
546  %add6 = add i16 %add5, %mul5
547  %add7 = add i16 %add6, %mul6
548  %add8 = add i16 %add7, %mul7
549
550  store i16 %add8, i16 addrspace(1)* %dst, align 4
551  ret void
552}
553
554; TODO: Remove the unnecessary instruction(that is zero-extending the
555; 2nd MAD) to have the pattern-recognizer to kick in.
556define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
557; GFX7-LABEL: udot8_acc8:
558; GFX7:       ; %bb.0: ; %entry
559; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
560; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
561; GFX7-NEXT:    s_mov_b32 s7, 0xf000
562; GFX7-NEXT:    s_mov_b32 s6, -1
563; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
564; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
565; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
566; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
567; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
568; GFX7-NEXT:    s_lshr_b32 s2, s0, 28
569; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40018
570; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40014
571; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40010
572; GFX7-NEXT:    s_bfe_u32 s18, s1, 0x4000c
573; GFX7-NEXT:    s_bfe_u32 s19, s1, 0x40008
574; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40004
575; GFX7-NEXT:    s_lshr_b32 s14, s1, 28
576; GFX7-NEXT:    s_and_b32 s1, s1, 15
577; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40018
578; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40014
579; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40010
580; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x4000c
581; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x40008
582; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40004
583; GFX7-NEXT:    s_and_b32 s0, s0, 15
584; GFX7-NEXT:    v_mov_b32_e32 v1, s1
585; GFX7-NEXT:    v_mov_b32_e32 v2, s20
586; GFX7-NEXT:    v_mov_b32_e32 v3, s19
587; GFX7-NEXT:    v_mov_b32_e32 v4, s18
588; GFX7-NEXT:    v_mov_b32_e32 v5, s17
589; GFX7-NEXT:    v_mov_b32_e32 v6, s16
590; GFX7-NEXT:    v_mov_b32_e32 v7, s15
591; GFX7-NEXT:    s_waitcnt vmcnt(0)
592; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
593; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v2, v0
594; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
595; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v4, v0
596; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v5, v0
597; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v6, v0
598; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v7, v0
599; GFX7-NEXT:    v_mov_b32_e32 v1, s14
600; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
601; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
602; GFX7-NEXT:    s_endpgm
603;
604; GFX8-LABEL: udot8_acc8:
605; GFX8:       ; %bb.0: ; %entry
606; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
607; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
608; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
609; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
610; GFX8-NEXT:    v_mov_b32_e32 v0, s0
611; GFX8-NEXT:    v_mov_b32_e32 v1, s1
612; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
613; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
614; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
615; GFX8-NEXT:    s_and_b32 s1, s2, 15
616; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40004
617; GFX8-NEXT:    v_mov_b32_e32 v3, s1
618; GFX8-NEXT:    v_mov_b32_e32 v4, s4
619; GFX8-NEXT:    s_and_b32 s1, s0, 15
620; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x40004
621; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40008
622; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x4000c
623; GFX8-NEXT:    v_mov_b32_e32 v5, s4
624; GFX8-NEXT:    s_bfe_u32 s6, s0, 0x40008
625; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40010
626; GFX8-NEXT:    v_mov_b32_e32 v6, s7
627; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x4000c
628; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40014
629; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x40010
630; GFX8-NEXT:    v_mov_b32_e32 v7, s8
631; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40018
632; GFX8-NEXT:    s_bfe_u32 s10, s0, 0x40014
633; GFX8-NEXT:    v_mov_b32_e32 v8, s9
634; GFX8-NEXT:    s_bfe_u32 s12, s0, 0x40018
635; GFX8-NEXT:    s_lshr_b32 s2, s2, 28
636; GFX8-NEXT:    v_mov_b32_e32 v9, s11
637; GFX8-NEXT:    s_lshr_b32 s0, s0, 28
638; GFX8-NEXT:    s_waitcnt vmcnt(0)
639; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
640; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
641; GFX8-NEXT:    v_and_b32_e32 v2, 0xff, v2
642; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
643; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
644; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
645; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
646; GFX8-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
647; GFX8-NEXT:    v_mov_b32_e32 v3, s2
648; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
649; GFX8-NEXT:    flat_store_byte v[0:1], v2
650; GFX8-NEXT:    s_endpgm
651;
652; GFX9-LABEL: udot8_acc8:
653; GFX9:       ; %bb.0: ; %entry
654; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
655; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
656; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
657; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
658; GFX9-NEXT:    v_mov_b32_e32 v0, s0
659; GFX9-NEXT:    v_mov_b32_e32 v1, s1
660; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
661; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
662; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
663; GFX9-NEXT:    s_and_b32 s1, s2, 15
664; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40004
665; GFX9-NEXT:    v_mov_b32_e32 v3, s1
666; GFX9-NEXT:    v_mov_b32_e32 v4, s4
667; GFX9-NEXT:    s_and_b32 s1, s0, 15
668; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40004
669; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40008
670; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x4000c
671; GFX9-NEXT:    v_mov_b32_e32 v5, s4
672; GFX9-NEXT:    s_bfe_u32 s6, s0, 0x40008
673; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40010
674; GFX9-NEXT:    v_mov_b32_e32 v6, s7
675; GFX9-NEXT:    s_bfe_u32 s4, s0, 0x4000c
676; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40014
677; GFX9-NEXT:    s_bfe_u32 s7, s0, 0x40010
678; GFX9-NEXT:    v_mov_b32_e32 v7, s8
679; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40018
680; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40014
681; GFX9-NEXT:    v_mov_b32_e32 v8, s9
682; GFX9-NEXT:    s_bfe_u32 s12, s0, 0x40018
683; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
684; GFX9-NEXT:    v_mov_b32_e32 v9, s11
685; GFX9-NEXT:    s_lshr_b32 s0, s0, 28
686; GFX9-NEXT:    s_waitcnt vmcnt(0)
687; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
688; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
689; GFX9-NEXT:    v_and_b32_e32 v2, 0xff, v2
690; GFX9-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
691; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
692; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
693; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
694; GFX9-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
695; GFX9-NEXT:    v_mov_b32_e32 v3, s2
696; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
697; GFX9-NEXT:    global_store_byte v[0:1], v2, off
698; GFX9-NEXT:    s_endpgm
699;
700; GFX9-DL-LABEL: udot8_acc8:
701; GFX9-DL:       ; %bb.0: ; %entry
702; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
703; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
704; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
705; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
706; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
707; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
708; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
709; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
710; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
711; GFX9-DL-NEXT:    s_and_b32 s1, s2, 15
712; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40004
713; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
714; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
715; GFX9-DL-NEXT:    s_and_b32 s1, s0, 15
716; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
717; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40008
718; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x4000c
719; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s4
720; GFX9-DL-NEXT:    s_bfe_u32 s6, s0, 0x40008
721; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40010
722; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s7
723; GFX9-DL-NEXT:    s_bfe_u32 s4, s0, 0x4000c
724; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40014
725; GFX9-DL-NEXT:    s_bfe_u32 s7, s0, 0x40010
726; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s8
727; GFX9-DL-NEXT:    s_bfe_u32 s11, s2, 0x40018
728; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40014
729; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s9
730; GFX9-DL-NEXT:    s_bfe_u32 s12, s0, 0x40018
731; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 28
732; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s11
733; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 28
734; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
735; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
736; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
737; GFX9-DL-NEXT:    v_and_b32_e32 v2, 0xff, v2
738; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
739; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
740; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
741; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
742; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
743; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
744; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
745; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
746; GFX9-DL-NEXT:    s_endpgm
747;
748; GFX10-DL-LABEL: udot8_acc8:
749; GFX10-DL:       ; %bb.0: ; %entry
750; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
751; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
752; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
753; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
754; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
755; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
756; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
757; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
758; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
759; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
760; GFX10-DL-NEXT:    s_and_b32 s2, s0, 15
761; GFX10-DL-NEXT:    s_and_b32 s4, s1, 15
762; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
763; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40004
764; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
765; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
766; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40008
767; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40008
768; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
769; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x4000c
770; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x4000c
771; GFX10-DL-NEXT:    v_and_b32_e32 v2, 0xff, v2
772; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
773; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40010
774; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40010
775; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
776; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
777; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40014
778; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
779; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
780; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40018
781; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
782; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
783; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
784; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
785; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
786; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
787; GFX10-DL-NEXT:    s_endpgm
788                                      <8 x i4> addrspace(1)* %src2,
789                                      i8 addrspace(1)* nocapture %dst) {
790entry:
791  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
792  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
793
794  %v1e0 = extractelement <8 x i4> %vec1, i64 0
795  %cv1e0 = zext i4 %v1e0 to i8
796  %v2e0 = extractelement <8 x i4> %vec2, i64 0
797  %cv2e0 = zext i4 %v2e0 to i8
798  %mul0 = mul nuw nsw i8 %cv1e0, %cv2e0
799
800  %v1e1 = extractelement <8 x i4> %vec1, i64 1
801  %cv1e1 = zext i4 %v1e1 to i8
802  %v2e1 = extractelement <8 x i4> %vec2, i64 1
803  %cv2e1 = zext i4 %v2e1 to i8
804  %mul1 = mul nuw nsw i8 %cv1e1, %cv2e1
805
806  %v1e2 = extractelement <8 x i4> %vec1, i64 2
807  %cv1e2 = zext i4 %v1e2 to i8
808  %v2e2 = extractelement <8 x i4> %vec2, i64 2
809  %cv2e2 = zext i4 %v2e2 to i8
810  %mul2 = mul nuw nsw i8 %cv1e2, %cv2e2
811
812  %v1e3 = extractelement <8 x i4> %vec1, i64 3
813  %cv1e3 = zext i4 %v1e3 to i8
814  %v2e3 = extractelement <8 x i4> %vec2, i64 3
815  %cv2e3 = zext i4 %v2e3 to i8
816  %mul3 = mul nuw nsw i8 %cv1e3, %cv2e3
817
818  %v1e4 = extractelement <8 x i4> %vec1, i64 4
819  %cv1e4 = zext i4 %v1e4 to i8
820  %v2e4 = extractelement <8 x i4> %vec2, i64 4
821  %cv2e4 = zext i4 %v2e4 to i8
822  %mul4 = mul nuw nsw i8 %cv1e4, %cv2e4
823
824  %v1e5 = extractelement <8 x i4> %vec1, i64 5
825  %cv1e5 = zext i4 %v1e5 to i8
826  %v2e5 = extractelement <8 x i4> %vec2, i64 5
827  %cv2e5 = zext i4 %v2e5 to i8
828  %mul5 = mul nuw nsw i8 %cv1e5, %cv2e5
829
830  %v1e6 = extractelement <8 x i4> %vec1, i64 6
831  %cv1e6 = zext i4 %v1e6 to i8
832  %v2e6 = extractelement <8 x i4> %vec2, i64 6
833  %cv2e6 = zext i4 %v2e6 to i8
834  %mul6 = mul nuw nsw i8 %cv1e6, %cv2e6
835
836  %v1e7 = extractelement <8 x i4> %vec1, i64 7
837  %cv1e7 = zext i4 %v1e7 to i8
838  %v2e7 = extractelement <8 x i4> %vec2, i64 7
839  %cv2e7 = zext i4 %v2e7 to i8
840  %mul7 = mul nuw nsw i8 %cv1e7, %cv2e7
841
842  %acc = load i8, i8 addrspace(1)* %dst, align 4
843  %add1 = add i8 %mul0, %acc
844  %add2 = add i8 %add1, %mul1
845  %add3 = add i8 %add2, %mul2
846  %add4 = add i8 %add3, %mul3
847  %add5 = add i8 %add4, %mul4
848  %add6 = add i8 %add5, %mul5
849  %add7 = add i8 %add6, %mul6
850  %add8 = add i8 %add7, %mul7
851
852  store i8 %add8, i8 addrspace(1)* %dst, align 4
853  ret void
854}
855
856; TODO: Remove the two unnecessary instructions(and+add after 2nd MAD)
857; to have the pattern-recognizer to kick in.
858define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
859; GFX7-LABEL: udot8_acc4:
860; GFX7:       ; %bb.0: ; %entry
861; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
862; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
863; GFX7-NEXT:    s_mov_b32 s7, 0xf000
864; GFX7-NEXT:    s_mov_b32 s6, -1
865; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
866; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
867; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
868; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
869; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
870; GFX7-NEXT:    s_lshr_b32 s2, s0, 28
871; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40018
872; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40014
873; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40010
874; GFX7-NEXT:    s_bfe_u32 s18, s1, 0x4000c
875; GFX7-NEXT:    s_bfe_u32 s19, s1, 0x40008
876; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40004
877; GFX7-NEXT:    s_lshr_b32 s14, s1, 28
878; GFX7-NEXT:    s_and_b32 s1, s1, 15
879; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40018
880; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40014
881; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40010
882; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x4000c
883; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x40008
884; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40004
885; GFX7-NEXT:    s_and_b32 s0, s0, 15
886; GFX7-NEXT:    v_mov_b32_e32 v1, s1
887; GFX7-NEXT:    v_mov_b32_e32 v2, s20
888; GFX7-NEXT:    v_mov_b32_e32 v3, s19
889; GFX7-NEXT:    v_mov_b32_e32 v4, s18
890; GFX7-NEXT:    v_mov_b32_e32 v5, s17
891; GFX7-NEXT:    v_mov_b32_e32 v6, s16
892; GFX7-NEXT:    v_mov_b32_e32 v7, s15
893; GFX7-NEXT:    s_waitcnt vmcnt(0)
894; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
895; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v2, v0
896; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
897; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v4, v0
898; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v5, v0
899; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v6, v0
900; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v7, v0
901; GFX7-NEXT:    v_mov_b32_e32 v1, s14
902; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
903; GFX7-NEXT:    v_and_b32_e32 v0, 15, v0
904; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
905; GFX7-NEXT:    s_endpgm
906;
907; GFX8-LABEL: udot8_acc4:
908; GFX8:       ; %bb.0: ; %entry
909; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
910; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
911; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
912; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
913; GFX8-NEXT:    v_mov_b32_e32 v0, s0
914; GFX8-NEXT:    v_mov_b32_e32 v1, s1
915; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
916; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
917; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
918; GFX8-NEXT:    s_and_b32 s1, s2, 15
919; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40004
920; GFX8-NEXT:    v_mov_b32_e32 v3, s1
921; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40008
922; GFX8-NEXT:    s_and_b32 s1, s0, 15
923; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x4000c
924; GFX8-NEXT:    v_mov_b32_e32 v4, s4
925; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x40004
926; GFX8-NEXT:    v_mov_b32_e32 v5, s7
927; GFX8-NEXT:    s_bfe_u32 s8, s0, 0x4000c
928; GFX8-NEXT:    v_mov_b32_e32 v6, s6
929; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x40008
930; GFX8-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
931; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40010
932; GFX8-NEXT:    v_and_b32_e32 v5, 15, v5
933; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40014
934; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x40010
935; GFX8-NEXT:    v_mov_b32_e32 v7, s6
936; GFX8-NEXT:    s_lshr_b32 s11, s2, 28
937; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x40018
938; GFX8-NEXT:    s_bfe_u32 s9, s0, 0x40014
939; GFX8-NEXT:    v_mov_b32_e32 v8, s8
940; GFX8-NEXT:    s_bfe_u32 s10, s0, 0x40018
941; GFX8-NEXT:    v_mov_b32_e32 v9, s2
942; GFX8-NEXT:    s_lshr_b32 s0, s0, 28
943; GFX8-NEXT:    s_waitcnt vmcnt(0)
944; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
945; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
946; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
947; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
948; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v5, v2
949; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
950; GFX8-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
951; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
952; GFX8-NEXT:    v_mov_b32_e32 v3, s11
953; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
954; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
955; GFX8-NEXT:    flat_store_byte v[0:1], v2
956; GFX8-NEXT:    s_endpgm
957;
958; GFX9-LABEL: udot8_acc4:
959; GFX9:       ; %bb.0: ; %entry
960; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
961; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
962; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
963; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
964; GFX9-NEXT:    v_mov_b32_e32 v0, s0
965; GFX9-NEXT:    v_mov_b32_e32 v1, s1
966; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
967; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
968; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
969; GFX9-NEXT:    s_and_b32 s1, s2, 15
970; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40004
971; GFX9-NEXT:    v_mov_b32_e32 v3, s1
972; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40008
973; GFX9-NEXT:    s_and_b32 s1, s0, 15
974; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x4000c
975; GFX9-NEXT:    v_mov_b32_e32 v4, s4
976; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40004
977; GFX9-NEXT:    v_mov_b32_e32 v5, s7
978; GFX9-NEXT:    s_bfe_u32 s8, s0, 0x4000c
979; GFX9-NEXT:    v_mov_b32_e32 v6, s6
980; GFX9-NEXT:    s_bfe_u32 s4, s0, 0x40008
981; GFX9-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
982; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40010
983; GFX9-NEXT:    v_and_b32_e32 v5, 15, v5
984; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
985; GFX9-NEXT:    s_bfe_u32 s7, s0, 0x40010
986; GFX9-NEXT:    v_mov_b32_e32 v7, s6
987; GFX9-NEXT:    s_lshr_b32 s11, s2, 28
988; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x40018
989; GFX9-NEXT:    s_bfe_u32 s9, s0, 0x40014
990; GFX9-NEXT:    v_mov_b32_e32 v8, s8
991; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40018
992; GFX9-NEXT:    v_mov_b32_e32 v9, s2
993; GFX9-NEXT:    s_lshr_b32 s0, s0, 28
994; GFX9-NEXT:    s_waitcnt vmcnt(0)
995; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
996; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
997; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
998; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
999; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
1000; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
1001; GFX9-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
1002; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
1003; GFX9-NEXT:    v_mov_b32_e32 v3, s11
1004; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
1005; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
1006; GFX9-NEXT:    global_store_byte v[0:1], v2, off
1007; GFX9-NEXT:    s_endpgm
1008;
1009; GFX9-DL-LABEL: udot8_acc4:
1010; GFX9-DL:       ; %bb.0: ; %entry
1011; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1012; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1013; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1014; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
1015; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1016; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1017; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
1018; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
1019; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1020; GFX9-DL-NEXT:    s_and_b32 s1, s2, 15
1021; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40004
1022; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
1023; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40008
1024; GFX9-DL-NEXT:    s_and_b32 s1, s0, 15
1025; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x4000c
1026; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
1027; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
1028; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s7
1029; GFX9-DL-NEXT:    s_bfe_u32 s8, s0, 0x4000c
1030; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s6
1031; GFX9-DL-NEXT:    s_bfe_u32 s4, s0, 0x40008
1032; GFX9-DL-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
1033; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40010
1034; GFX9-DL-NEXT:    v_and_b32_e32 v5, 15, v5
1035; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40014
1036; GFX9-DL-NEXT:    s_bfe_u32 s7, s0, 0x40010
1037; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s6
1038; GFX9-DL-NEXT:    s_lshr_b32 s11, s2, 28
1039; GFX9-DL-NEXT:    s_bfe_u32 s2, s2, 0x40018
1040; GFX9-DL-NEXT:    s_bfe_u32 s9, s0, 0x40014
1041; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s8
1042; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40018
1043; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s2
1044; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 28
1045; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
1046; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
1047; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
1048; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
1049; GFX9-DL-NEXT:    v_and_b32_e32 v2, 15, v2
1050; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
1051; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
1052; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
1053; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
1054; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s11
1055; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
1056; GFX9-DL-NEXT:    v_and_b32_e32 v2, 15, v2
1057; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
1058; GFX9-DL-NEXT:    s_endpgm
1059;
1060; GFX10-DL-LABEL: udot8_acc4:
1061; GFX10-DL:       ; %bb.0: ; %entry
1062; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1063; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1064; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1065; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1066; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1067; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1068; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
1069; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
1070; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
1071; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1072; GFX10-DL-NEXT:    s_and_b32 s2, s0, 15
1073; GFX10-DL-NEXT:    s_and_b32 s4, s1, 15
1074; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
1075; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40004
1076; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x40008
1077; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
1078; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
1079; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40008
1080; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x4000c
1081; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
1082; GFX10-DL-NEXT:    s_bfe_u32 s5, s1, 0x4000c
1083; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40014
1084; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s7, v2
1085; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v3, s4, s5
1086; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40010
1087; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40010
1088; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
1089; GFX10-DL-NEXT:    v_and_b32_e32 v2, 15, v2
1090; GFX10-DL-NEXT:    v_and_b32_e32 v3, 15, v3
1091; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
1092; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
1093; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
1094; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40018
1095; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
1096; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
1097; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
1098; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
1099; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
1100; GFX10-DL-NEXT:    v_and_b32_e32 v2, 15, v2
1101; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
1102; GFX10-DL-NEXT:    s_endpgm
1103                                      <8 x i4> addrspace(1)* %src2,
1104                                      i4 addrspace(1)* nocapture %dst) {
1105entry:
1106  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1107  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1108
1109  %v1e0 = extractelement <8 x i4> %vec1, i64 0
1110  %v2e0 = extractelement <8 x i4> %vec2, i64 0
1111  %mul0 = mul nuw nsw i4 %v1e0, %v2e0
1112
1113  %v1e1 = extractelement <8 x i4> %vec1, i64 1
1114  %v2e1 = extractelement <8 x i4> %vec2, i64 1
1115  %mul1 = mul nuw nsw i4 %v1e1, %v2e1
1116
1117  %v1e2 = extractelement <8 x i4> %vec1, i64 2
1118  %v2e2 = extractelement <8 x i4> %vec2, i64 2
1119  %mul2 = mul nuw nsw i4 %v1e2, %v2e2
1120
1121  %v1e3 = extractelement <8 x i4> %vec1, i64 3
1122  %v2e3 = extractelement <8 x i4> %vec2, i64 3
1123  %mul3 = mul nuw nsw i4 %v1e3, %v2e3
1124
1125  %v1e4 = extractelement <8 x i4> %vec1, i64 4
1126  %v2e4 = extractelement <8 x i4> %vec2, i64 4
1127  %mul4 = mul nuw nsw i4 %v1e4, %v2e4
1128
1129  %v1e5 = extractelement <8 x i4> %vec1, i64 5
1130  %v2e5 = extractelement <8 x i4> %vec2, i64 5
1131  %mul5 = mul nuw nsw i4 %v1e5, %v2e5
1132
1133  %v1e6 = extractelement <8 x i4> %vec1, i64 6
1134  %v2e6 = extractelement <8 x i4> %vec2, i64 6
1135  %mul6 = mul nuw nsw i4 %v1e6, %v2e6
1136
1137  %v1e7 = extractelement <8 x i4> %vec1, i64 7
1138  %v2e7 = extractelement <8 x i4> %vec2, i64 7
1139  %mul7 = mul nuw nsw i4 %v1e7, %v2e7
1140
1141  %acc = load i4, i4 addrspace(1)* %dst, align 4
1142  %add1 = add i4 %mul0, %acc
1143  %add2 = add i4 %add1, %mul1
1144  %add3 = add i4 %add2, %mul2
1145  %add4 = add i4 %add3, %mul3
1146  %add5 = add i4 %add4, %mul4
1147  %add6 = add i4 %add5, %mul5
1148  %add7 = add i4 %add6, %mul6
1149  %add8 = add i4 %add7, %mul7
1150
1151  store i4 %add8, i4 addrspace(1)* %dst, align 4
1152  ret void
1153}
1154
1155; TODO: Currently, permutation of udot8 is turned off due to a huge increase
1156; in the compile time.
1157define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %src1,
1158; GFX7-LABEL: udot8_CommutationInsideMAD:
1159; GFX7:       ; %bb.0: ; %entry
1160; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
1161; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
1162; GFX7-NEXT:    s_mov_b32 s7, 0xf000
1163; GFX7-NEXT:    s_mov_b32 s6, -1
1164; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1165; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
1166; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
1167; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
1168; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1169; GFX7-NEXT:    s_lshr_b32 s2, s0, 28
1170; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40018
1171; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40014
1172; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40010
1173; GFX7-NEXT:    s_bfe_u32 s18, s1, 0x4000c
1174; GFX7-NEXT:    s_bfe_u32 s19, s1, 0x40008
1175; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40004
1176; GFX7-NEXT:    s_lshr_b32 s14, s1, 28
1177; GFX7-NEXT:    s_and_b32 s1, s1, 15
1178; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40018
1179; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40014
1180; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40010
1181; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x4000c
1182; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x40008
1183; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40004
1184; GFX7-NEXT:    s_and_b32 s0, s0, 15
1185; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1186; GFX7-NEXT:    v_mov_b32_e32 v2, s20
1187; GFX7-NEXT:    v_mov_b32_e32 v3, s19
1188; GFX7-NEXT:    v_mov_b32_e32 v4, s18
1189; GFX7-NEXT:    v_mov_b32_e32 v5, s17
1190; GFX7-NEXT:    v_mov_b32_e32 v6, s16
1191; GFX7-NEXT:    v_mov_b32_e32 v7, s15
1192; GFX7-NEXT:    s_waitcnt vmcnt(0)
1193; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
1194; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v2, v0
1195; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
1196; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v4, v0
1197; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v5, v0
1198; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v6, v0
1199; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v7, v0
1200; GFX7-NEXT:    v_mov_b32_e32 v1, s14
1201; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1202; GFX7-NEXT:    v_and_b32_e32 v0, 15, v0
1203; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
1204; GFX7-NEXT:    s_endpgm
1205;
1206; GFX8-LABEL: udot8_CommutationInsideMAD:
1207; GFX8:       ; %bb.0: ; %entry
1208; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1209; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1210; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1211; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
1212; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1213; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1214; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
1215; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
1216; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1217; GFX8-NEXT:    s_and_b32 s1, s2, 15
1218; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40004
1219; GFX8-NEXT:    v_mov_b32_e32 v3, s1
1220; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40008
1221; GFX8-NEXT:    s_and_b32 s1, s0, 15
1222; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x4000c
1223; GFX8-NEXT:    v_mov_b32_e32 v4, s4
1224; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x40004
1225; GFX8-NEXT:    v_mov_b32_e32 v5, s7
1226; GFX8-NEXT:    s_bfe_u32 s8, s0, 0x4000c
1227; GFX8-NEXT:    v_mov_b32_e32 v6, s6
1228; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x40008
1229; GFX8-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
1230; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40010
1231; GFX8-NEXT:    v_and_b32_e32 v5, 15, v5
1232; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40014
1233; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x40010
1234; GFX8-NEXT:    v_mov_b32_e32 v7, s6
1235; GFX8-NEXT:    s_lshr_b32 s11, s2, 28
1236; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x40018
1237; GFX8-NEXT:    s_bfe_u32 s9, s0, 0x40014
1238; GFX8-NEXT:    v_mov_b32_e32 v8, s8
1239; GFX8-NEXT:    s_bfe_u32 s10, s0, 0x40018
1240; GFX8-NEXT:    v_mov_b32_e32 v9, s2
1241; GFX8-NEXT:    s_lshr_b32 s0, s0, 28
1242; GFX8-NEXT:    s_waitcnt vmcnt(0)
1243; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
1244; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
1245; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
1246; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
1247; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
1248; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
1249; GFX8-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
1250; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
1251; GFX8-NEXT:    v_mov_b32_e32 v3, s11
1252; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
1253; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
1254; GFX8-NEXT:    flat_store_byte v[0:1], v2
1255; GFX8-NEXT:    s_endpgm
1256;
1257; GFX9-LABEL: udot8_CommutationInsideMAD:
1258; GFX9:       ; %bb.0: ; %entry
1259; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1260; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1261; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1262; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
1263; GFX9-NEXT:    v_mov_b32_e32 v0, s0
1264; GFX9-NEXT:    v_mov_b32_e32 v1, s1
1265; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
1266; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
1267; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1268; GFX9-NEXT:    s_and_b32 s1, s2, 15
1269; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40004
1270; GFX9-NEXT:    v_mov_b32_e32 v3, s1
1271; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40008
1272; GFX9-NEXT:    s_and_b32 s1, s0, 15
1273; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x4000c
1274; GFX9-NEXT:    v_mov_b32_e32 v4, s4
1275; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40004
1276; GFX9-NEXT:    v_mov_b32_e32 v5, s7
1277; GFX9-NEXT:    s_bfe_u32 s8, s0, 0x4000c
1278; GFX9-NEXT:    v_mov_b32_e32 v6, s6
1279; GFX9-NEXT:    s_bfe_u32 s4, s0, 0x40008
1280; GFX9-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
1281; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40010
1282; GFX9-NEXT:    v_and_b32_e32 v5, 15, v5
1283; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
1284; GFX9-NEXT:    s_bfe_u32 s7, s0, 0x40010
1285; GFX9-NEXT:    v_mov_b32_e32 v7, s6
1286; GFX9-NEXT:    s_lshr_b32 s11, s2, 28
1287; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x40018
1288; GFX9-NEXT:    s_bfe_u32 s9, s0, 0x40014
1289; GFX9-NEXT:    v_mov_b32_e32 v8, s8
1290; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40018
1291; GFX9-NEXT:    v_mov_b32_e32 v9, s2
1292; GFX9-NEXT:    s_lshr_b32 s0, s0, 28
1293; GFX9-NEXT:    s_waitcnt vmcnt(0)
1294; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
1295; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
1296; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
1297; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
1298; GFX9-NEXT:    v_add_u32_e32 v2, v5, v2
1299; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
1300; GFX9-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
1301; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
1302; GFX9-NEXT:    v_mov_b32_e32 v3, s11
1303; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
1304; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
1305; GFX9-NEXT:    global_store_byte v[0:1], v2, off
1306; GFX9-NEXT:    s_endpgm
1307;
1308; GFX9-DL-LABEL: udot8_CommutationInsideMAD:
1309; GFX9-DL:       ; %bb.0: ; %entry
1310; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1311; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1312; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1313; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
1314; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1315; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1316; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
1317; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
1318; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1319; GFX9-DL-NEXT:    s_and_b32 s1, s2, 15
1320; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40004
1321; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
1322; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40008
1323; GFX9-DL-NEXT:    s_and_b32 s1, s0, 15
1324; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x4000c
1325; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
1326; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
1327; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s7
1328; GFX9-DL-NEXT:    s_bfe_u32 s8, s0, 0x4000c
1329; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s6
1330; GFX9-DL-NEXT:    s_bfe_u32 s4, s0, 0x40008
1331; GFX9-DL-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
1332; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40010
1333; GFX9-DL-NEXT:    v_and_b32_e32 v5, 15, v5
1334; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40014
1335; GFX9-DL-NEXT:    s_bfe_u32 s7, s0, 0x40010
1336; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s6
1337; GFX9-DL-NEXT:    s_lshr_b32 s11, s2, 28
1338; GFX9-DL-NEXT:    s_bfe_u32 s2, s2, 0x40018
1339; GFX9-DL-NEXT:    s_bfe_u32 s9, s0, 0x40014
1340; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s8
1341; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40018
1342; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s2
1343; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 28
1344; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
1345; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
1346; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
1347; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
1348; GFX9-DL-NEXT:    v_and_b32_e32 v2, 15, v2
1349; GFX9-DL-NEXT:    v_add_u32_e32 v2, v5, v2
1350; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
1351; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
1352; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
1353; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s11
1354; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
1355; GFX9-DL-NEXT:    v_and_b32_e32 v2, 15, v2
1356; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
1357; GFX9-DL-NEXT:    s_endpgm
1358;
1359; GFX10-DL-LABEL: udot8_CommutationInsideMAD:
1360; GFX10-DL:       ; %bb.0: ; %entry
1361; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1362; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1363; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1364; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1365; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1366; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1367; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
1368; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
1369; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
1370; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1371; GFX10-DL-NEXT:    s_and_b32 s2, s0, 15
1372; GFX10-DL-NEXT:    s_and_b32 s4, s1, 15
1373; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
1374; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40004
1375; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x40008
1376; GFX10-DL-NEXT:    s_bfe_u32 s8, s1, 0x4000c
1377; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
1378; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
1379; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x4000c
1380; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40008
1381; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
1382; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v3, s4, s8
1383; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40010
1384; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
1385; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40014
1386; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s7, v2
1387; GFX10-DL-NEXT:    v_and_b32_e32 v3, 15, v3
1388; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40010
1389; GFX10-DL-NEXT:    v_and_b32_e32 v2, 15, v2
1390; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
1391; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
1392; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
1393; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40018
1394; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
1395; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
1396; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
1397; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
1398; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
1399; GFX10-DL-NEXT:    v_and_b32_e32 v2, 15, v2
1400; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
1401; GFX10-DL-NEXT:    s_endpgm
1402                                                      <8 x i4> addrspace(1)* %src2,
1403                                                      i4 addrspace(1)* nocapture %dst) {
1404entry:
1405  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1406  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1407
1408  %v1e0 = extractelement <8 x i4> %vec1, i64 0
1409  %v2e0 = extractelement <8 x i4> %vec2, i64 0
1410  %mul0 = mul nuw nsw i4 %v1e0, %v2e0
1411
1412  %v1e1 = extractelement <8 x i4> %vec1, i64 1
1413  %v2e1 = extractelement <8 x i4> %vec2, i64 1
1414  %mul1 = mul nuw nsw i4 %v1e1, %v2e1
1415
1416  %v1e2 = extractelement <8 x i4> %vec1, i64 2
1417  %v2e2 = extractelement <8 x i4> %vec2, i64 2
1418  %mul2 = mul nuw nsw i4 %v1e2, %v2e2
1419
1420  %v1e3 = extractelement <8 x i4> %vec1, i64 3
1421  %v2e3 = extractelement <8 x i4> %vec2, i64 3
1422  %mul3 = mul nuw nsw i4 %v1e3, %v2e3
1423
1424  %v1e4 = extractelement <8 x i4> %vec1, i64 4
1425  %v2e4 = extractelement <8 x i4> %vec2, i64 4
1426  %mul4 = mul nuw nsw i4 %v1e4, %v2e4
1427
1428  %v1e5 = extractelement <8 x i4> %vec1, i64 5
1429  %v2e5 = extractelement <8 x i4> %vec2, i64 5
1430  %mul5 = mul nuw nsw i4 %v1e5, %v2e5
1431
1432  %v1e6 = extractelement <8 x i4> %vec1, i64 6
1433  %v2e6 = extractelement <8 x i4> %vec2, i64 6
1434  %mul6 = mul nuw nsw i4 %v1e6, %v2e6
1435
1436  %v1e7 = extractelement <8 x i4> %vec1, i64 7
1437  %v2e7 = extractelement <8 x i4> %vec2, i64 7
1438  %mul7 = mul nuw nsw i4 %v1e7, %v2e7
1439
1440  %acc = load i4, i4 addrspace(1)* %dst, align 4
1441  %add1 = add i4 %mul0, %acc
1442  %add2 = add i4 %mul1, %add1
1443  %add3 = add i4 %mul2, %add2
1444  %add4 = add i4 %mul3, %add3
1445  %add5 = add i4 %mul4, %add4
1446  %add6 = add i4 %mul5, %add5
1447  %add7 = add i4 %mul6, %add6
1448  %add8 = add i4 %mul7, %add7
1449
1450  store i4 %add8, i4 addrspace(1)* %dst, align 4
1451  ret void
1452}
1453
1454define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
1455; GFX7-LABEL: udot8_multiuses_mul1:
1456; GFX7:       ; %bb.0: ; %entry
1457; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
1458; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
1459; GFX7-NEXT:    s_mov_b32 s7, 0xf000
1460; GFX7-NEXT:    s_mov_b32 s6, -1
1461; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1462; GFX7-NEXT:    s_load_dword s10, s[10:11], 0x0
1463; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
1464; GFX7-NEXT:    s_load_dword s21, s[4:5], 0x0
1465; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1466; GFX7-NEXT:    s_bfe_u32 s20, s10, 0x40004
1467; GFX7-NEXT:    s_lshr_b32 s11, s10, 28
1468; GFX7-NEXT:    s_bfe_u32 s15, s10, 0x40018
1469; GFX7-NEXT:    s_bfe_u32 s16, s10, 0x40014
1470; GFX7-NEXT:    s_bfe_u32 s17, s10, 0x40010
1471; GFX7-NEXT:    s_bfe_u32 s18, s10, 0x4000c
1472; GFX7-NEXT:    s_bfe_u32 s19, s10, 0x40008
1473; GFX7-NEXT:    s_and_b32 s10, s10, 15
1474; GFX7-NEXT:    s_lshr_b32 s1, s0, 28
1475; GFX7-NEXT:    s_bfe_u32 s2, s0, 0x40018
1476; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40014
1477; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40010
1478; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x4000c
1479; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40008
1480; GFX7-NEXT:    s_bfe_u32 s14, s0, 0x40004
1481; GFX7-NEXT:    s_and_b32 s0, s0, 15
1482; GFX7-NEXT:    v_mov_b32_e32 v0, s10
1483; GFX7-NEXT:    v_mov_b32_e32 v1, s21
1484; GFX7-NEXT:    v_mad_u32_u24 v1, s0, v0, v1
1485; GFX7-NEXT:    v_mov_b32_e32 v2, s20
1486; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v0, v1
1487; GFX7-NEXT:    v_mad_u32_u24 v1, s14, v2, v1
1488; GFX7-NEXT:    v_mov_b32_e32 v2, s19
1489; GFX7-NEXT:    v_mad_u32_u24 v1, s13, v2, v1
1490; GFX7-NEXT:    v_mov_b32_e32 v2, s18
1491; GFX7-NEXT:    v_mad_u32_u24 v1, s12, v2, v1
1492; GFX7-NEXT:    v_mov_b32_e32 v2, s17
1493; GFX7-NEXT:    v_mad_u32_u24 v1, s9, v2, v1
1494; GFX7-NEXT:    v_mov_b32_e32 v2, s16
1495; GFX7-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
1496; GFX7-NEXT:    v_mov_b32_e32 v2, s15
1497; GFX7-NEXT:    v_mad_u32_u24 v1, s2, v2, v1
1498; GFX7-NEXT:    v_mov_b32_e32 v2, s11
1499; GFX7-NEXT:    v_mad_u32_u24 v1, s1, v2, v1
1500; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
1501; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1502; GFX7-NEXT:    s_endpgm
1503;
1504; GFX8-LABEL: udot8_multiuses_mul1:
1505; GFX8:       ; %bb.0: ; %entry
1506; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1507; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1508; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1509; GFX8-NEXT:    s_load_dword s6, s[6:7], 0x0
1510; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
1511; GFX8-NEXT:    s_load_dword s19, s[0:1], 0x0
1512; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1513; GFX8-NEXT:    s_bfe_u32 s18, s6, 0x40004
1514; GFX8-NEXT:    s_lshr_b32 s7, s6, 28
1515; GFX8-NEXT:    s_bfe_u32 s13, s6, 0x40018
1516; GFX8-NEXT:    s_bfe_u32 s14, s6, 0x40014
1517; GFX8-NEXT:    s_bfe_u32 s15, s6, 0x40010
1518; GFX8-NEXT:    s_bfe_u32 s16, s6, 0x4000c
1519; GFX8-NEXT:    s_bfe_u32 s17, s6, 0x40008
1520; GFX8-NEXT:    s_and_b32 s6, s6, 15
1521; GFX8-NEXT:    s_lshr_b32 s4, s2, 28
1522; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x40018
1523; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40014
1524; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40010
1525; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x4000c
1526; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40008
1527; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40004
1528; GFX8-NEXT:    s_and_b32 s2, s2, 15
1529; GFX8-NEXT:    v_mov_b32_e32 v0, s6
1530; GFX8-NEXT:    v_mov_b32_e32 v1, s19
1531; GFX8-NEXT:    v_mad_u32_u24 v1, s2, v0, v1
1532; GFX8-NEXT:    v_mov_b32_e32 v2, s18
1533; GFX8-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
1534; GFX8-NEXT:    v_mad_u32_u24 v1, s12, v2, v1
1535; GFX8-NEXT:    v_mov_b32_e32 v2, s17
1536; GFX8-NEXT:    v_mad_u32_u24 v1, s11, v2, v1
1537; GFX8-NEXT:    v_mov_b32_e32 v2, s16
1538; GFX8-NEXT:    v_mad_u32_u24 v1, s10, v2, v1
1539; GFX8-NEXT:    v_mov_b32_e32 v2, s15
1540; GFX8-NEXT:    v_mad_u32_u24 v1, s9, v2, v1
1541; GFX8-NEXT:    v_mov_b32_e32 v2, s14
1542; GFX8-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
1543; GFX8-NEXT:    v_mov_b32_e32 v2, s13
1544; GFX8-NEXT:    v_mad_u32_u24 v1, s5, v2, v1
1545; GFX8-NEXT:    v_mov_b32_e32 v2, s7
1546; GFX8-NEXT:    v_mad_u32_u24 v1, s4, v2, v1
1547; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v1, v0
1548; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1549; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1550; GFX8-NEXT:    flat_store_dword v[0:1], v2
1551; GFX8-NEXT:    s_endpgm
1552;
1553; GFX9-LABEL: udot8_multiuses_mul1:
1554; GFX9:       ; %bb.0: ; %entry
1555; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1556; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1557; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1558; GFX9-NEXT:    s_load_dword s6, s[6:7], 0x0
1559; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
1560; GFX9-NEXT:    s_load_dword s19, s[0:1], 0x0
1561; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1562; GFX9-NEXT:    s_bfe_u32 s18, s6, 0x40004
1563; GFX9-NEXT:    s_lshr_b32 s7, s6, 28
1564; GFX9-NEXT:    s_bfe_u32 s13, s6, 0x40018
1565; GFX9-NEXT:    s_bfe_u32 s14, s6, 0x40014
1566; GFX9-NEXT:    s_bfe_u32 s15, s6, 0x40010
1567; GFX9-NEXT:    s_bfe_u32 s16, s6, 0x4000c
1568; GFX9-NEXT:    s_bfe_u32 s17, s6, 0x40008
1569; GFX9-NEXT:    s_and_b32 s6, s6, 15
1570; GFX9-NEXT:    s_lshr_b32 s4, s2, 28
1571; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40018
1572; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
1573; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40010
1574; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x4000c
1575; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40008
1576; GFX9-NEXT:    s_bfe_u32 s12, s2, 0x40004
1577; GFX9-NEXT:    s_and_b32 s2, s2, 15
1578; GFX9-NEXT:    v_mov_b32_e32 v0, s6
1579; GFX9-NEXT:    v_mov_b32_e32 v1, s19
1580; GFX9-NEXT:    v_mad_u32_u24 v1, s2, v0, v1
1581; GFX9-NEXT:    v_mov_b32_e32 v2, s18
1582; GFX9-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
1583; GFX9-NEXT:    v_mad_u32_u24 v1, s12, v2, v1
1584; GFX9-NEXT:    v_mov_b32_e32 v2, s17
1585; GFX9-NEXT:    v_mad_u32_u24 v1, s11, v2, v1
1586; GFX9-NEXT:    v_mov_b32_e32 v2, s16
1587; GFX9-NEXT:    v_mad_u32_u24 v1, s10, v2, v1
1588; GFX9-NEXT:    v_mov_b32_e32 v2, s15
1589; GFX9-NEXT:    v_mad_u32_u24 v1, s9, v2, v1
1590; GFX9-NEXT:    v_mov_b32_e32 v2, s14
1591; GFX9-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
1592; GFX9-NEXT:    v_mov_b32_e32 v2, s13
1593; GFX9-NEXT:    v_mad_u32_u24 v1, s5, v2, v1
1594; GFX9-NEXT:    v_mov_b32_e32 v2, s7
1595; GFX9-NEXT:    v_mad_u32_u24 v1, s4, v2, v1
1596; GFX9-NEXT:    v_add_u32_e32 v2, v0, v1
1597; GFX9-NEXT:    v_mov_b32_e32 v0, s0
1598; GFX9-NEXT:    v_mov_b32_e32 v1, s1
1599; GFX9-NEXT:    global_store_dword v[0:1], v2, off
1600; GFX9-NEXT:    s_endpgm
1601;
1602; GFX9-DL-LABEL: udot8_multiuses_mul1:
1603; GFX9-DL:       ; %bb.0: ; %entry
1604; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1605; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1606; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1607; GFX9-DL-NEXT:    s_load_dword s6, s[6:7], 0x0
1608; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1609; GFX9-DL-NEXT:    s_load_dword s19, s[0:1], 0x0
1610; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1611; GFX9-DL-NEXT:    s_bfe_u32 s18, s6, 0x40004
1612; GFX9-DL-NEXT:    s_lshr_b32 s7, s6, 28
1613; GFX9-DL-NEXT:    s_bfe_u32 s13, s6, 0x40018
1614; GFX9-DL-NEXT:    s_bfe_u32 s14, s6, 0x40014
1615; GFX9-DL-NEXT:    s_bfe_u32 s15, s6, 0x40010
1616; GFX9-DL-NEXT:    s_bfe_u32 s16, s6, 0x4000c
1617; GFX9-DL-NEXT:    s_bfe_u32 s17, s6, 0x40008
1618; GFX9-DL-NEXT:    s_and_b32 s6, s6, 15
1619; GFX9-DL-NEXT:    s_lshr_b32 s4, s2, 28
1620; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x40018
1621; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40014
1622; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40010
1623; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x4000c
1624; GFX9-DL-NEXT:    s_bfe_u32 s11, s2, 0x40008
1625; GFX9-DL-NEXT:    s_bfe_u32 s12, s2, 0x40004
1626; GFX9-DL-NEXT:    s_and_b32 s2, s2, 15
1627; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
1628; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s19
1629; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s2, v0, v1
1630; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s18
1631; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
1632; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s12, v2, v1
1633; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s17
1634; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s11, v2, v1
1635; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s16
1636; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s10, v2, v1
1637; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s15
1638; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s9, v2, v1
1639; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s14
1640; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
1641; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s13
1642; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s5, v2, v1
1643; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s7
1644; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s4, v2, v1
1645; GFX9-DL-NEXT:    v_add_u32_e32 v2, v0, v1
1646; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1647; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1648; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1649; GFX9-DL-NEXT:    s_endpgm
1650;
1651; GFX10-DL-LABEL: udot8_multiuses_mul1:
1652; GFX10-DL:       ; %bb.0: ; %entry
1653; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1654; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1655; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1656; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1657; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1658; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1659; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1660; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1661; GFX10-DL-NEXT:    s_and_b32 s6, s2, 15
1662; GFX10-DL-NEXT:    s_and_b32 s7, s4, 15
1663; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
1664; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40004
1665; GFX10-DL-NEXT:    s_bfe_u32 s8, s4, 0x40004
1666; GFX10-DL-NEXT:    s_bfe_u32 s9, s2, 0x40008
1667; GFX10-DL-NEXT:    s_bfe_u32 s10, s4, 0x40008
1668; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s7, v0
1669; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s5, s8, v0
1670; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x4000c
1671; GFX10-DL-NEXT:    s_bfe_u32 s8, s4, 0x4000c
1672; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s7, v0
1673; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s9, s10, v1
1674; GFX10-DL-NEXT:    s_bfe_u32 s9, s2, 0x40010
1675; GFX10-DL-NEXT:    s_bfe_u32 s10, s4, 0x40010
1676; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s5, s8, v1
1677; GFX10-DL-NEXT:    s_bfe_u32 s5, s2, 0x40014
1678; GFX10-DL-NEXT:    s_bfe_u32 s8, s4, 0x40014
1679; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s9, s10, v1
1680; GFX10-DL-NEXT:    s_bfe_u32 s9, s2, 0x40018
1681; GFX10-DL-NEXT:    s_bfe_u32 s10, s4, 0x40018
1682; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 28
1683; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 28
1684; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s5, s8, v1
1685; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s9, s10, v1
1686; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s2, s4, v1
1687; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v0, v1
1688; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1689; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1690; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1691; GFX10-DL-NEXT:    s_endpgm
1692                                                <8 x i4> addrspace(1)* %src2,
1693                                                i32 addrspace(1)* nocapture %dst) {
1694entry:
1695  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1696  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1697
1698  %v1e0 = extractelement <8 x i4> %vec1, i64 0
1699  %cv1e0 = zext i4 %v1e0 to i32
1700  %v2e0 = extractelement <8 x i4> %vec2, i64 0
1701  %cv2e0 = zext i4 %v2e0 to i32
1702  %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
1703
1704  %v1e1 = extractelement <8 x i4> %vec1, i64 1
1705  %cv1e1 = zext i4 %v1e1 to i32
1706  %v2e1 = extractelement <8 x i4> %vec2, i64 1
1707  %cv2e1 = zext i4 %v2e1 to i32
1708  %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
1709
1710  %v1e2 = extractelement <8 x i4> %vec1, i64 2
1711  %cv1e2 = zext i4 %v1e2 to i32
1712  %v2e2 = extractelement <8 x i4> %vec2, i64 2
1713  %cv2e2 = zext i4 %v2e2 to i32
1714  %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
1715
1716  %v1e3 = extractelement <8 x i4> %vec1, i64 3
1717  %cv1e3 = zext i4 %v1e3 to i32
1718  %v2e3 = extractelement <8 x i4> %vec2, i64 3
1719  %cv2e3 = zext i4 %v2e3 to i32
1720  %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
1721
1722  %v1e4 = extractelement <8 x i4> %vec1, i64 4
1723  %cv1e4 = zext i4 %v1e4 to i32
1724  %v2e4 = extractelement <8 x i4> %vec2, i64 4
1725  %cv2e4 = zext i4 %v2e4 to i32
1726  %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
1727
1728  %v1e5 = extractelement <8 x i4> %vec1, i64 5
1729  %cv1e5 = zext i4 %v1e5 to i32
1730  %v2e5 = extractelement <8 x i4> %vec2, i64 5
1731  %cv2e5 = zext i4 %v2e5 to i32
1732  %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
1733
1734  %v1e6 = extractelement <8 x i4> %vec1, i64 6
1735  %cv1e6 = zext i4 %v1e6 to i32
1736  %v2e6 = extractelement <8 x i4> %vec2, i64 6
1737  %cv2e6 = zext i4 %v2e6 to i32
1738  %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
1739
1740  %v1e7 = extractelement <8 x i4> %vec1, i64 7
1741  %cv1e7 = zext i4 %v1e7 to i32
1742  %v2e7 = extractelement <8 x i4> %vec2, i64 7
1743  %cv2e7 = zext i4 %v2e7 to i32
1744  %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
1745
1746  %acc = load i32, i32 addrspace(1)* %dst, align 4
1747  %add1 = add i32 %mul0, %acc
1748  %add = add i32  %mul0, %add1
1749  %add2 = add i32 %add1, %mul1
1750  %add3 = add i32 %add2, %mul2
1751  %add4 = add i32 %add3, %mul3
1752  %add5 = add i32 %add4, %mul4
1753  %add6 = add i32 %add5, %mul5
1754  %add7 = add i32 %add6, %mul6
1755  %add8 = add i32 %add7, %mul7
1756
1757  %res = add i32 %add, %add8
1758  store i32 %res, i32 addrspace(1)* %dst, align 4
1759  ret void
1760}
1761
1762define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
1763; GFX7-LABEL: udot8_acc32_vecMul:
1764; GFX7:       ; %bb.0: ; %entry
1765; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
1766; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
1767; GFX7-NEXT:    s_mov_b32 s7, 0xf000
1768; GFX7-NEXT:    s_mov_b32 s6, -1
1769; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1770; GFX7-NEXT:    s_load_dword s10, s[10:11], 0x0
1771; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
1772; GFX7-NEXT:    s_load_dword s21, s[4:5], 0x0
1773; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1774; GFX7-NEXT:    s_lshr_b32 s11, s10, 28
1775; GFX7-NEXT:    s_bfe_u32 s15, s10, 0x40018
1776; GFX7-NEXT:    s_bfe_u32 s16, s10, 0x40014
1777; GFX7-NEXT:    s_bfe_u32 s17, s10, 0x40010
1778; GFX7-NEXT:    s_bfe_u32 s18, s10, 0x4000c
1779; GFX7-NEXT:    s_bfe_u32 s19, s10, 0x40008
1780; GFX7-NEXT:    s_bfe_u32 s20, s10, 0x40004
1781; GFX7-NEXT:    s_and_b32 s10, s10, 15
1782; GFX7-NEXT:    s_lshr_b32 s1, s0, 28
1783; GFX7-NEXT:    s_bfe_u32 s2, s0, 0x40018
1784; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40014
1785; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40010
1786; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x4000c
1787; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40008
1788; GFX7-NEXT:    s_bfe_u32 s14, s0, 0x40004
1789; GFX7-NEXT:    s_and_b32 s0, s0, 15
1790; GFX7-NEXT:    v_mov_b32_e32 v0, s10
1791; GFX7-NEXT:    v_mov_b32_e32 v1, s21
1792; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v0, v1
1793; GFX7-NEXT:    v_mov_b32_e32 v1, s20
1794; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v1, v0
1795; GFX7-NEXT:    v_mov_b32_e32 v1, s19
1796; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v1, v0
1797; GFX7-NEXT:    v_mov_b32_e32 v1, s18
1798; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
1799; GFX7-NEXT:    v_mov_b32_e32 v1, s17
1800; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
1801; GFX7-NEXT:    v_mov_b32_e32 v1, s16
1802; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
1803; GFX7-NEXT:    v_mov_b32_e32 v1, s15
1804; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1805; GFX7-NEXT:    v_mov_b32_e32 v1, s11
1806; GFX7-NEXT:    v_mad_u32_u24 v0, s1, v1, v0
1807; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1808; GFX7-NEXT:    s_endpgm
1809;
1810; GFX8-LABEL: udot8_acc32_vecMul:
1811; GFX8:       ; %bb.0: ; %entry
1812; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1813; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1814; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1815; GFX8-NEXT:    s_load_dword s6, s[6:7], 0x0
1816; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
1817; GFX8-NEXT:    s_load_dword s19, s[0:1], 0x0
1818; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1819; GFX8-NEXT:    s_lshr_b32 s7, s6, 28
1820; GFX8-NEXT:    s_bfe_u32 s13, s6, 0x40018
1821; GFX8-NEXT:    s_bfe_u32 s14, s6, 0x40014
1822; GFX8-NEXT:    s_bfe_u32 s15, s6, 0x40010
1823; GFX8-NEXT:    s_bfe_u32 s16, s6, 0x4000c
1824; GFX8-NEXT:    s_bfe_u32 s17, s6, 0x40008
1825; GFX8-NEXT:    s_bfe_u32 s18, s6, 0x40004
1826; GFX8-NEXT:    s_and_b32 s6, s6, 15
1827; GFX8-NEXT:    s_lshr_b32 s4, s2, 28
1828; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x40018
1829; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40014
1830; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40010
1831; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x4000c
1832; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40008
1833; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40004
1834; GFX8-NEXT:    s_and_b32 s2, s2, 15
1835; GFX8-NEXT:    v_mov_b32_e32 v0, s6
1836; GFX8-NEXT:    v_mov_b32_e32 v1, s19
1837; GFX8-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
1838; GFX8-NEXT:    v_mov_b32_e32 v1, s18
1839; GFX8-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
1840; GFX8-NEXT:    v_mov_b32_e32 v1, s17
1841; GFX8-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
1842; GFX8-NEXT:    v_mov_b32_e32 v1, s16
1843; GFX8-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
1844; GFX8-NEXT:    v_mov_b32_e32 v1, s15
1845; GFX8-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
1846; GFX8-NEXT:    v_mov_b32_e32 v1, s14
1847; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
1848; GFX8-NEXT:    v_mov_b32_e32 v1, s13
1849; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1850; GFX8-NEXT:    v_mov_b32_e32 v1, s7
1851; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1852; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1853; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1854; GFX8-NEXT:    flat_store_dword v[0:1], v2
1855; GFX8-NEXT:    s_endpgm
1856;
1857; GFX9-LABEL: udot8_acc32_vecMul:
1858; GFX9:       ; %bb.0: ; %entry
1859; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1860; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1861; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1862; GFX9-NEXT:    s_load_dword s6, s[6:7], 0x0
1863; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
1864; GFX9-NEXT:    s_load_dword s19, s[0:1], 0x0
1865; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1866; GFX9-NEXT:    s_lshr_b32 s7, s6, 28
1867; GFX9-NEXT:    s_bfe_u32 s13, s6, 0x40018
1868; GFX9-NEXT:    s_bfe_u32 s14, s6, 0x40014
1869; GFX9-NEXT:    s_bfe_u32 s15, s6, 0x40010
1870; GFX9-NEXT:    s_bfe_u32 s16, s6, 0x4000c
1871; GFX9-NEXT:    s_bfe_u32 s17, s6, 0x40008
1872; GFX9-NEXT:    s_bfe_u32 s18, s6, 0x40004
1873; GFX9-NEXT:    s_and_b32 s6, s6, 15
1874; GFX9-NEXT:    s_lshr_b32 s4, s2, 28
1875; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40018
1876; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
1877; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40010
1878; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x4000c
1879; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40008
1880; GFX9-NEXT:    s_bfe_u32 s12, s2, 0x40004
1881; GFX9-NEXT:    s_and_b32 s2, s2, 15
1882; GFX9-NEXT:    v_mov_b32_e32 v0, s6
1883; GFX9-NEXT:    v_mov_b32_e32 v1, s19
1884; GFX9-NEXT:    v_mad_u32_u24 v0, s2, v0, v1
1885; GFX9-NEXT:    v_mov_b32_e32 v1, s18
1886; GFX9-NEXT:    v_mad_u32_u24 v0, s12, v1, v0
1887; GFX9-NEXT:    v_mov_b32_e32 v1, s17
1888; GFX9-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
1889; GFX9-NEXT:    v_mov_b32_e32 v1, s16
1890; GFX9-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
1891; GFX9-NEXT:    v_mov_b32_e32 v1, s15
1892; GFX9-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
1893; GFX9-NEXT:    v_mov_b32_e32 v1, s14
1894; GFX9-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
1895; GFX9-NEXT:    v_mov_b32_e32 v1, s13
1896; GFX9-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1897; GFX9-NEXT:    v_mov_b32_e32 v1, s7
1898; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1899; GFX9-NEXT:    v_mov_b32_e32 v0, s0
1900; GFX9-NEXT:    v_mov_b32_e32 v1, s1
1901; GFX9-NEXT:    global_store_dword v[0:1], v2, off
1902; GFX9-NEXT:    s_endpgm
1903;
1904; GFX9-DL-LABEL: udot8_acc32_vecMul:
1905; GFX9-DL:       ; %bb.0: ; %entry
1906; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1907; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1908; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1909; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
1910; GFX9-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
1911; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
1912; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1913; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1914; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1915; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
1916; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s6
1917; GFX9-DL-NEXT:    v_dot8_u32_u4 v2, s4, v2, v3
1918; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1919; GFX9-DL-NEXT:    s_endpgm
1920;
1921; GFX10-DL-LABEL: udot8_acc32_vecMul:
1922; GFX10-DL:       ; %bb.0: ; %entry
1923; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1924; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1925; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1926; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1927; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
1928; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
1929; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
1930; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1931; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
1932; GFX10-DL-NEXT:    v_dot8_u32_u4 v2, s4, s5, v0
1933; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1934; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1935; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1936; GFX10-DL-NEXT:    s_endpgm
1937                                              <8 x i4> addrspace(1)* %src2,
1938                                              i32 addrspace(1)* nocapture %dst) {
1939entry:
1940  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1941  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1942
1943  %cvec1 = zext <8 x i4> %vec1 to <8 x i32>
1944  %cvec2 = zext <8 x i4> %vec2 to <8 x i32>
1945
1946  %mul = mul <8 x i32> %cvec1, %cvec2
1947  %mul0 = extractelement <8 x i32> %mul, i64 0
1948  %mul1 = extractelement <8 x i32> %mul, i64 1
1949  %mul2 = extractelement <8 x i32> %mul, i64 2
1950  %mul3 = extractelement <8 x i32> %mul, i64 3
1951  %mul4 = extractelement <8 x i32> %mul, i64 4
1952  %mul5 = extractelement <8 x i32> %mul, i64 5
1953  %mul6 = extractelement <8 x i32> %mul, i64 6
1954  %mul7 = extractelement <8 x i32> %mul, i64 7
1955
1956  %acc = load i32, i32 addrspace(1)* %dst, align 4
1957  %add1 = add i32 %mul0, %acc
1958  %add2 = add i32 %add1, %mul1
1959  %add3 = add i32 %add2, %mul2
1960  %add4 = add i32 %add3, %mul3
1961  %add5 = add i32 %add4, %mul4
1962  %add6 = add i32 %add5, %mul5
1963  %add7 = add i32 %add6, %mul6
1964  %add8 = add i32 %add7, %mul7
1965
1966  store i32 %add8, i32 addrspace(1)* %dst, align 4
1967  ret void
1968}
1969
1970; TODO: Clean up the code(by default pk_mad_I16 should be generated), then
1971; support the pattern.
1972define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
1973; GFX7-LABEL: udot8_acc16_vecMul:
1974; GFX7:       ; %bb.0: ; %entry
1975; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
1976; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
1977; GFX7-NEXT:    s_mov_b32 s7, 0xf000
1978; GFX7-NEXT:    s_mov_b32 s6, -1
1979; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1980; GFX7-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
1981; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
1982; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
1983; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1984; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x40004
1985; GFX7-NEXT:    s_bfe_u32 s18, s1, 0x40004
1986; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x4000c
1987; GFX7-NEXT:    v_mov_b32_e32 v4, s18
1988; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40018
1989; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40014
1990; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40010
1991; GFX7-NEXT:    s_and_b32 s19, s1, 15
1992; GFX7-NEXT:    s_lshr_b32 s14, s1, 28
1993; GFX7-NEXT:    s_bfe_u32 s1, s1, 0x40008
1994; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x4000c
1995; GFX7-NEXT:    v_mov_b32_e32 v2, s20
1996; GFX7-NEXT:    v_mul_u32_u24_e32 v2, s13, v2
1997; GFX7-NEXT:    v_mul_u32_u24_e32 v4, s11, v4
1998; GFX7-NEXT:    s_lshr_b32 s2, s0, 28
1999; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40018
2000; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40014
2001; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40010
2002; GFX7-NEXT:    s_and_b32 s12, s0, 15
2003; GFX7-NEXT:    v_mov_b32_e32 v3, s19
2004; GFX7-NEXT:    s_bfe_u32 s0, s0, 0x40008
2005; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2006; GFX7-NEXT:    v_mul_u32_u24_e32 v1, s0, v1
2007; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
2008; GFX7-NEXT:    v_mul_u32_u24_e32 v3, s12, v3
2009; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
2010; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
2011; GFX7-NEXT:    v_or_b32_e32 v2, v3, v4
2012; GFX7-NEXT:    v_alignbit_b32 v3, v1, v2, 16
2013; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
2014; GFX7-NEXT:    v_mov_b32_e32 v5, s17
2015; GFX7-NEXT:    v_mov_b32_e32 v6, s16
2016; GFX7-NEXT:    v_mov_b32_e32 v7, s15
2017; GFX7-NEXT:    s_waitcnt vmcnt(0)
2018; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
2019; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
2020; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
2021; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
2022; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v5, v0
2023; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v6, v0
2024; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v7, v0
2025; GFX7-NEXT:    v_mov_b32_e32 v1, s14
2026; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
2027; GFX7-NEXT:    buffer_store_short v0, off, s[4:7], 0
2028; GFX7-NEXT:    s_endpgm
2029;
2030; GFX8-LABEL: udot8_acc16_vecMul:
2031; GFX8:       ; %bb.0: ; %entry
2032; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2033; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2034; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2035; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
2036; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2037; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2038; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
2039; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
2040; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2041; GFX8-NEXT:    s_and_b32 s1, s2, 15
2042; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40004
2043; GFX8-NEXT:    v_mov_b32_e32 v3, s1
2044; GFX8-NEXT:    v_mov_b32_e32 v4, s4
2045; GFX8-NEXT:    s_and_b32 s1, s0, 15
2046; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x40004
2047; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40008
2048; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x4000c
2049; GFX8-NEXT:    v_mov_b32_e32 v5, s4
2050; GFX8-NEXT:    s_bfe_u32 s6, s0, 0x40008
2051; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40010
2052; GFX8-NEXT:    v_mov_b32_e32 v6, s7
2053; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x4000c
2054; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x40014
2055; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x40010
2056; GFX8-NEXT:    v_mov_b32_e32 v7, s8
2057; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40018
2058; GFX8-NEXT:    s_bfe_u32 s10, s0, 0x40014
2059; GFX8-NEXT:    v_mov_b32_e32 v8, s9
2060; GFX8-NEXT:    s_bfe_u32 s12, s0, 0x40018
2061; GFX8-NEXT:    s_lshr_b32 s2, s2, 28
2062; GFX8-NEXT:    v_mov_b32_e32 v9, s11
2063; GFX8-NEXT:    s_lshr_b32 s0, s0, 28
2064; GFX8-NEXT:    s_waitcnt vmcnt(0)
2065; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
2066; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
2067; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff, v2
2068; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
2069; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
2070; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
2071; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v8, v2
2072; GFX8-NEXT:    v_mad_u32_u24 v2, s12, v9, v2
2073; GFX8-NEXT:    v_mov_b32_e32 v3, s2
2074; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
2075; GFX8-NEXT:    flat_store_short v[0:1], v2
2076; GFX8-NEXT:    s_endpgm
2077;
2078; GFX9-LABEL: udot8_acc16_vecMul:
2079; GFX9:       ; %bb.0: ; %entry
2080; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2081; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2082; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2083; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
2084; GFX9-NEXT:    v_mov_b32_e32 v0, s0
2085; GFX9-NEXT:    v_mov_b32_e32 v1, s1
2086; GFX9-NEXT:    global_load_ushort v2, v[0:1], off
2087; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
2088; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2089; GFX9-NEXT:    s_and_b32 s1, s2, 15
2090; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40004
2091; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
2092; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40018
2093; GFX9-NEXT:    s_lshr_b32 s13, s2, 28
2094; GFX9-NEXT:    s_and_b32 s4, s0, 15
2095; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40004
2096; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
2097; GFX9-NEXT:    v_mov_b32_e32 v3, s1
2098; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
2099; GFX9-NEXT:    s_bfe_u32 s1, s0, 0x40008
2100; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x4000c
2101; GFX9-NEXT:    s_bfe_u32 s9, s0, 0x40010
2102; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40014
2103; GFX9-NEXT:    s_bfe_u32 s12, s0, 0x40018
2104; GFX9-NEXT:    s_lshr_b32 s0, s0, 28
2105; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40008
2106; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x4000c
2107; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40010
2108; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x40014
2109; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s12, s0
2110; GFX9-NEXT:    v_mov_b32_e32 v4, s11
2111; GFX9-NEXT:    v_pk_mul_lo_u16 v4, s0, v4
2112; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s8, s2
2113; GFX9-NEXT:    v_mov_b32_e32 v5, s0
2114; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s6, s7
2115; GFX9-NEXT:    v_pk_mul_lo_u16 v3, s4, v3
2116; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
2117; GFX9-NEXT:    v_mov_b32_e32 v6, s0
2118; GFX9-NEXT:    v_pk_mul_lo_u16 v6, s1, v6
2119; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s9, s10
2120; GFX9-NEXT:    v_pk_mul_lo_u16 v5, s2, v5
2121; GFX9-NEXT:    s_waitcnt vmcnt(0)
2122; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
2123; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2124; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
2125; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2126; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
2127; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2128; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
2129; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2130; GFX9-NEXT:    global_store_short v[0:1], v2, off
2131; GFX9-NEXT:    s_endpgm
2132;
2133; GFX9-DL-LABEL: udot8_acc16_vecMul:
2134; GFX9-DL:       ; %bb.0: ; %entry
2135; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2136; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2137; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2138; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
2139; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2140; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2141; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
2142; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
2143; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2144; GFX9-DL-NEXT:    s_and_b32 s1, s2, 15
2145; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40004
2146; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
2147; GFX9-DL-NEXT:    s_bfe_u32 s11, s2, 0x40018
2148; GFX9-DL-NEXT:    s_lshr_b32 s13, s2, 28
2149; GFX9-DL-NEXT:    s_and_b32 s4, s0, 15
2150; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
2151; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
2152; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
2153; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
2154; GFX9-DL-NEXT:    s_bfe_u32 s1, s0, 0x40008
2155; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x4000c
2156; GFX9-DL-NEXT:    s_bfe_u32 s9, s0, 0x40010
2157; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40014
2158; GFX9-DL-NEXT:    s_bfe_u32 s12, s0, 0x40018
2159; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 28
2160; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40008
2161; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x4000c
2162; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40010
2163; GFX9-DL-NEXT:    s_bfe_u32 s2, s2, 0x40014
2164; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s12, s0
2165; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s11
2166; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v4, s0, v4
2167; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s8, s2
2168; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s0
2169; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s6, s7
2170; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, s4, v3
2171; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
2172; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s0
2173; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v6, s1, v6
2174; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s9, s10
2175; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v5, s2, v5
2176; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2177; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
2178; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2179; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
2180; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2181; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
2182; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2183; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v4
2184; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2185; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
2186; GFX9-DL-NEXT:    s_endpgm
2187;
2188; GFX10-DL-LABEL: udot8_acc16_vecMul:
2189; GFX10-DL:       ; %bb.0: ; %entry
2190; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2191; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2192; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2193; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2194; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2195; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2196; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
2197; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
2198; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
2199; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2200; GFX10-DL-NEXT:    s_and_b32 s2, s0, 15
2201; GFX10-DL-NEXT:    s_bfe_u32 s6, s0, 0x40004
2202; GFX10-DL-NEXT:    s_and_b32 s4, s1, 15
2203; GFX10-DL-NEXT:    s_bfe_u32 s5, s1, 0x40004
2204; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x4000c
2205; GFX10-DL-NEXT:    s_bfe_u32 s8, s0, 0x4000c
2206; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
2207; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40008
2208; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
2209; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40008
2210; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, s2, s4
2211; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s4, s6, s7
2212; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s5, s5, s8
2213; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40010
2214; GFX10-DL-NEXT:    s_bfe_u32 s6, s0, 0x40014
2215; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x40010
2216; GFX10-DL-NEXT:    s_bfe_u32 s8, s1, 0x40014
2217; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, s5, s4
2218; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x40018
2219; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
2220; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
2221; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s5, s7, s8
2222; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40018
2223; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
2224; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s4, s0
2225; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s1, s6, s1
2226; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
2227; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
2228; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2229; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, s2, s5
2230; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0
2231; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2232; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, s0, s1
2233; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
2234; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2235; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v4
2236; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2237; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
2238; GFX10-DL-NEXT:    s_endpgm
2239                                              <8 x i4> addrspace(1)* %src2,
2240                                              i16 addrspace(1)* nocapture %dst) {
2241entry:
2242  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
2243  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
2244
2245  %cvec1 = zext <8 x i4> %vec1 to <8 x i16>
2246  %cvec2 = zext <8 x i4> %vec2 to <8 x i16>
2247
2248  %mul = mul <8 x i16> %cvec1, %cvec2
2249  %mul0 = extractelement <8 x i16> %mul, i64 0
2250  %mul1 = extractelement <8 x i16> %mul, i64 1
2251  %mul2 = extractelement <8 x i16> %mul, i64 2
2252  %mul3 = extractelement <8 x i16> %mul, i64 3
2253  %mul4 = extractelement <8 x i16> %mul, i64 4
2254  %mul5 = extractelement <8 x i16> %mul, i64 5
2255  %mul6 = extractelement <8 x i16> %mul, i64 6
2256  %mul7 = extractelement <8 x i16> %mul, i64 7
2257
2258  %acc = load i16, i16 addrspace(1)* %dst, align 4
2259  %add1 = add i16 %mul0, %acc
2260  %add2 = add i16 %add1, %mul1
2261  %add3 = add i16 %add2, %mul2
2262  %add4 = add i16 %add3, %mul3
2263  %add5 = add i16 %add4, %mul4
2264  %add6 = add i16 %add5, %mul5
2265  %add7 = add i16 %add6, %mul6
2266  %add8 = add i16 %add7, %mul7
2267
2268  store i16 %add8, i16 addrspace(1)* %dst, align 4
2269  ret void
2270}
2271
2272; TODO: Cleanup the code to generate MAD; pattern should be recognized then.
2273define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
2274; GFX7-LABEL: udot8_acc8_vecMul:
2275; GFX7:       ; %bb.0: ; %entry
2276; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
2277; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
2278; GFX7-NEXT:    s_mov_b32 s7, 0xf000
2279; GFX7-NEXT:    s_mov_b32 s6, -1
2280; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2281; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
2282; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
2283; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
2284; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2285; GFX7-NEXT:    s_bfe_u32 s2, s0, 0x4000c
2286; GFX7-NEXT:    s_bfe_u32 s14, s1, 0x4000c
2287; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40004
2288; GFX7-NEXT:    s_lshr_b32 s18, s1, 28
2289; GFX7-NEXT:    v_mov_b32_e32 v8, s14
2290; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40008
2291; GFX7-NEXT:    s_and_b32 s17, s1, 15
2292; GFX7-NEXT:    s_bfe_u32 s19, s1, 0x40018
2293; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40014
2294; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40004
2295; GFX7-NEXT:    v_mov_b32_e32 v6, s16
2296; GFX7-NEXT:    s_lshr_b32 s11, s0, 28
2297; GFX7-NEXT:    v_mov_b32_e32 v4, s18
2298; GFX7-NEXT:    v_mul_u32_u24_e32 v4, s11, v4
2299; GFX7-NEXT:    v_mul_u32_u24_e32 v6, s9, v6
2300; GFX7-NEXT:    v_mul_u32_u24_e32 v8, s2, v8
2301; GFX7-NEXT:    s_bfe_u32 s1, s1, 0x40010
2302; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40008
2303; GFX7-NEXT:    v_mov_b32_e32 v7, s15
2304; GFX7-NEXT:    s_and_b32 s10, s0, 15
2305; GFX7-NEXT:    v_mov_b32_e32 v5, s17
2306; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x40018
2307; GFX7-NEXT:    v_mov_b32_e32 v3, s19
2308; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40014
2309; GFX7-NEXT:    v_mov_b32_e32 v2, s20
2310; GFX7-NEXT:    v_mul_u32_u24_e32 v2, s13, v2
2311; GFX7-NEXT:    s_bfe_u32 s0, s0, 0x40010
2312; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2313; GFX7-NEXT:    v_mul_u32_u24_e32 v3, s12, v3
2314; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
2315; GFX7-NEXT:    v_mul_u32_u24_e32 v5, s10, v5
2316; GFX7-NEXT:    v_mul_u32_u24_e32 v7, s8, v7
2317; GFX7-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
2318; GFX7-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
2319; GFX7-NEXT:    v_or_b32_e32 v3, v3, v4
2320; GFX7-NEXT:    v_or_b32_e32 v4, v5, v6
2321; GFX7-NEXT:    v_or_b32_e32 v5, v7, v8
2322; GFX7-NEXT:    v_mul_u32_u24_e32 v9, s0, v1
2323; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
2324; GFX7-NEXT:    v_or_b32_e32 v2, v9, v2
2325; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
2326; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
2327; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
2328; GFX7-NEXT:    v_or_b32_e32 v3, v4, v5
2329; GFX7-NEXT:    v_alignbit_b32 v4, v2, v3, 8
2330; GFX7-NEXT:    v_alignbit_b32 v5, v2, v3, 16
2331; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 24, v3
2332; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 8, v2
2333; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
2334; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
2335; GFX7-NEXT:    s_waitcnt vmcnt(0)
2336; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
2337; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
2338; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
2339; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
2340; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
2341; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v7
2342; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
2343; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
2344; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
2345; GFX7-NEXT:    s_endpgm
2346;
2347; GFX8-LABEL: udot8_acc8_vecMul:
2348; GFX8:       ; %bb.0: ; %entry
2349; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2350; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2351; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2352; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2353; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2354; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
2355; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
2356; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
2357; GFX8-NEXT:    s_mov_b32 s0, 0xffff
2358; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2359; GFX8-NEXT:    s_bfe_u32 s8, s1, 0x40004
2360; GFX8-NEXT:    s_bfe_u32 s10, s1, 0x4000c
2361; GFX8-NEXT:    s_bfe_u32 s15, s2, 0x40004
2362; GFX8-NEXT:    s_and_b32 s16, s2, 15
2363; GFX8-NEXT:    s_bfe_u32 s17, s2, 0x4000c
2364; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x40014
2365; GFX8-NEXT:    s_lshr_b32 s6, s1, 28
2366; GFX8-NEXT:    s_bfe_u32 s11, s2, 0x40014
2367; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40010
2368; GFX8-NEXT:    s_lshr_b32 s13, s2, 28
2369; GFX8-NEXT:    s_bfe_u32 s14, s2, 0x40018
2370; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x40008
2371; GFX8-NEXT:    s_and_b32 s9, s1, 15
2372; GFX8-NEXT:    v_mov_b32_e32 v4, s17
2373; GFX8-NEXT:    v_mov_b32_e32 v5, s10
2374; GFX8-NEXT:    v_mov_b32_e32 v6, s16
2375; GFX8-NEXT:    v_mov_b32_e32 v7, s15
2376; GFX8-NEXT:    v_mov_b32_e32 v8, s8
2377; GFX8-NEXT:    v_mul_u32_u24_sdwa v4, v5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2378; GFX8-NEXT:    v_mul_u32_u24_e32 v5, s9, v6
2379; GFX8-NEXT:    v_mul_u32_u24_sdwa v6, v8, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2380; GFX8-NEXT:    s_bfe_u32 s5, s1, 0x40010
2381; GFX8-NEXT:    s_bfe_u32 s7, s1, 0x40018
2382; GFX8-NEXT:    v_mov_b32_e32 v9, s14
2383; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x40008
2384; GFX8-NEXT:    v_mov_b32_e32 v3, s2
2385; GFX8-NEXT:    v_mov_b32_e32 v10, s13
2386; GFX8-NEXT:    v_mov_b32_e32 v11, s6
2387; GFX8-NEXT:    v_mov_b32_e32 v12, s12
2388; GFX8-NEXT:    v_mov_b32_e32 v13, s11
2389; GFX8-NEXT:    v_mov_b32_e32 v14, s4
2390; GFX8-NEXT:    v_mul_u32_u24_e32 v3, s1, v3
2391; GFX8-NEXT:    v_or_b32_e32 v5, v5, v6
2392; GFX8-NEXT:    v_mul_u32_u24_e32 v7, s7, v9
2393; GFX8-NEXT:    v_mul_u32_u24_sdwa v8, v11, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2394; GFX8-NEXT:    v_mul_u32_u24_e32 v9, s5, v12
2395; GFX8-NEXT:    v_mul_u32_u24_sdwa v10, v14, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2396; GFX8-NEXT:    v_and_b32_e32 v5, s0, v5
2397; GFX8-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2398; GFX8-NEXT:    v_or_b32_e32 v9, v9, v10
2399; GFX8-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2400; GFX8-NEXT:    v_and_b32_e32 v4, s0, v9
2401; GFX8-NEXT:    v_or_b32_e32 v3, v5, v3
2402; GFX8-NEXT:    v_or_b32_e32 v6, v4, v7
2403; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 8, v3
2404; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 8, v6
2405; GFX8-NEXT:    s_waitcnt vmcnt(0)
2406; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
2407; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v7, v2
2408; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
2409; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
2410; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
2411; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v8, v2
2412; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
2413; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
2414; GFX8-NEXT:    flat_store_byte v[0:1], v2
2415; GFX8-NEXT:    s_endpgm
2416;
2417; GFX9-LABEL: udot8_acc8_vecMul:
2418; GFX9:       ; %bb.0: ; %entry
2419; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2420; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2421; GFX9-NEXT:    s_mov_b32 s2, 0xffff
2422; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2423; GFX9-NEXT:    v_mov_b32_e32 v0, s0
2424; GFX9-NEXT:    v_mov_b32_e32 v1, s1
2425; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
2426; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
2427; GFX9-NEXT:    s_load_dword s1, s[6:7], 0x0
2428; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2429; GFX9-NEXT:    s_bfe_u32 s4, s0, 0x40010
2430; GFX9-NEXT:    s_bfe_u32 s11, s1, 0x40010
2431; GFX9-NEXT:    s_bfe_u32 s12, s1, 0x40014
2432; GFX9-NEXT:    s_bfe_u32 s13, s1, 0x40018
2433; GFX9-NEXT:    s_lshr_b32 s14, s1, 28
2434; GFX9-NEXT:    s_and_b32 s15, s1, 15
2435; GFX9-NEXT:    s_bfe_u32 s16, s1, 0x40004
2436; GFX9-NEXT:    s_bfe_u32 s17, s1, 0x40008
2437; GFX9-NEXT:    v_mov_b32_e32 v3, s11
2438; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x4000c
2439; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40014
2440; GFX9-NEXT:    v_mov_b32_e32 v4, s12
2441; GFX9-NEXT:    s_bfe_u32 s6, s0, 0x40018
2442; GFX9-NEXT:    v_mov_b32_e32 v5, s13
2443; GFX9-NEXT:    s_lshr_b32 s7, s0, 28
2444; GFX9-NEXT:    v_mov_b32_e32 v6, s14
2445; GFX9-NEXT:    s_and_b32 s8, s0, 15
2446; GFX9-NEXT:    v_mov_b32_e32 v7, s15
2447; GFX9-NEXT:    s_bfe_u32 s9, s0, 0x40004
2448; GFX9-NEXT:    v_mov_b32_e32 v8, s16
2449; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40008
2450; GFX9-NEXT:    v_mov_b32_e32 v9, s17
2451; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x4000c
2452; GFX9-NEXT:    v_mov_b32_e32 v10, s1
2453; GFX9-NEXT:    v_mul_lo_u16_e32 v3, s4, v3
2454; GFX9-NEXT:    v_mul_lo_u16_sdwa v4, s5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2455; GFX9-NEXT:    v_mul_lo_u16_e32 v5, s6, v5
2456; GFX9-NEXT:    v_mul_lo_u16_sdwa v6, s7, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2457; GFX9-NEXT:    v_mul_lo_u16_e32 v7, s8, v7
2458; GFX9-NEXT:    v_mul_lo_u16_sdwa v8, s9, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2459; GFX9-NEXT:    v_or_b32_e32 v3, v3, v4
2460; GFX9-NEXT:    v_or_b32_sdwa v4, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2461; GFX9-NEXT:    v_or_b32_e32 v5, v7, v8
2462; GFX9-NEXT:    v_mul_lo_u16_e32 v9, s10, v9
2463; GFX9-NEXT:    v_mul_lo_u16_sdwa v10, s0, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2464; GFX9-NEXT:    v_and_b32_e32 v5, s2, v5
2465; GFX9-NEXT:    v_or_b32_sdwa v6, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2466; GFX9-NEXT:    v_or_b32_e32 v6, v5, v6
2467; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 8, v6
2468; GFX9-NEXT:    v_and_b32_e32 v3, s2, v3
2469; GFX9-NEXT:    v_or_b32_e32 v4, v3, v4
2470; GFX9-NEXT:    s_waitcnt vmcnt(0)
2471; GFX9-NEXT:    v_add_u32_e32 v2, v5, v2
2472; GFX9-NEXT:    v_add_u32_e32 v2, v2, v7
2473; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
2474; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2475; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
2476; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
2477; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
2478; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2479; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2480; GFX9-NEXT:    global_store_byte v[0:1], v2, off
2481; GFX9-NEXT:    s_endpgm
2482;
2483; GFX9-DL-LABEL: udot8_acc8_vecMul:
2484; GFX9-DL:       ; %bb.0: ; %entry
2485; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2486; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2487; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
2488; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2489; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2490; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2491; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
2492; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
2493; GFX9-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
2494; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2495; GFX9-DL-NEXT:    s_bfe_u32 s4, s0, 0x40010
2496; GFX9-DL-NEXT:    s_bfe_u32 s11, s1, 0x40010
2497; GFX9-DL-NEXT:    s_bfe_u32 s12, s1, 0x40014
2498; GFX9-DL-NEXT:    s_bfe_u32 s13, s1, 0x40018
2499; GFX9-DL-NEXT:    s_lshr_b32 s14, s1, 28
2500; GFX9-DL-NEXT:    s_and_b32 s15, s1, 15
2501; GFX9-DL-NEXT:    s_bfe_u32 s16, s1, 0x40004
2502; GFX9-DL-NEXT:    s_bfe_u32 s17, s1, 0x40008
2503; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s11
2504; GFX9-DL-NEXT:    s_bfe_u32 s1, s1, 0x4000c
2505; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
2506; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s12
2507; GFX9-DL-NEXT:    s_bfe_u32 s6, s0, 0x40018
2508; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s13
2509; GFX9-DL-NEXT:    s_lshr_b32 s7, s0, 28
2510; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s14
2511; GFX9-DL-NEXT:    s_and_b32 s8, s0, 15
2512; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s15
2513; GFX9-DL-NEXT:    s_bfe_u32 s9, s0, 0x40004
2514; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s16
2515; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40008
2516; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s17
2517; GFX9-DL-NEXT:    s_bfe_u32 s0, s0, 0x4000c
2518; GFX9-DL-NEXT:    v_mov_b32_e32 v10, s1
2519; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v3, s4, v3
2520; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, s5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2521; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v5, s6, v5
2522; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v6, s7, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2523; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v7, s8, v7
2524; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v8, s9, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2525; GFX9-DL-NEXT:    v_or_b32_e32 v3, v3, v4
2526; GFX9-DL-NEXT:    v_or_b32_sdwa v4, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2527; GFX9-DL-NEXT:    v_or_b32_e32 v5, v7, v8
2528; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v9, s10, v9
2529; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v10, s0, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2530; GFX9-DL-NEXT:    v_and_b32_e32 v5, s2, v5
2531; GFX9-DL-NEXT:    v_or_b32_sdwa v6, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2532; GFX9-DL-NEXT:    v_or_b32_e32 v6, v5, v6
2533; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v7, 8, v6
2534; GFX9-DL-NEXT:    v_and_b32_e32 v3, s2, v3
2535; GFX9-DL-NEXT:    v_or_b32_e32 v4, v3, v4
2536; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2537; GFX9-DL-NEXT:    v_add_u32_e32 v2, v5, v2
2538; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v7
2539; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
2540; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2541; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v3
2542; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
2543; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v3
2544; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2545; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2546; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
2547; GFX9-DL-NEXT:    s_endpgm
2548;
2549; GFX10-DL-LABEL: udot8_acc8_vecMul:
2550; GFX10-DL:       ; %bb.0: ; %entry
2551; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2552; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2553; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
2554; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2555; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2556; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2557; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2558; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
2559; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
2560; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
2561; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2562; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x40004
2563; GFX10-DL-NEXT:    s_bfe_u32 s5, s1, 0x40004
2564; GFX10-DL-NEXT:    s_and_b32 s6, s0, 15
2565; GFX10-DL-NEXT:    s_and_b32 s8, s1, 15
2566; GFX10-DL-NEXT:    s_bfe_u32 s7, s0, 0x4000c
2567; GFX10-DL-NEXT:    s_bfe_u32 s9, s1, 0x4000c
2568; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v3, s4, s5
2569; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x40008
2570; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, s6, s8
2571; GFX10-DL-NEXT:    s_bfe_u32 s5, s1, 0x40008
2572; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s7, s9
2573; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v3
2574; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x40014
2575; GFX10-DL-NEXT:    s_bfe_u32 s6, s0, 0x40018
2576; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, s4, s5
2577; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 8, v5
2578; GFX10-DL-NEXT:    v_or_b32_e32 v3, v4, v3
2579; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
2580; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x40010
2581; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
2582; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2583; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
2584; GFX10-DL-NEXT:    s_bfe_u32 s8, s1, 0x40010
2585; GFX10-DL-NEXT:    s_lshr_b32 s9, s1, 28
2586; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s5, s7
2587; GFX10-DL-NEXT:    s_bfe_u32 s1, s1, 0x40018
2588; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v4
2589; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, s4, s8
2590; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, s0, s9
2591; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v8, 8, v4
2592; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v7, 8, v7
2593; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
2594; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
2595; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v5
2596; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s6, s1
2597; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v8
2598; GFX10-DL-NEXT:    v_or_b32_e32 v3, v6, v3
2599; GFX10-DL-NEXT:    v_or_b32_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2600; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
2601; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
2602; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2603; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v5
2604; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
2605; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
2606; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
2607; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2608; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2609; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
2610; GFX10-DL-NEXT:    s_endpgm
2611                                             <8 x i4> addrspace(1)* %src2,
2612                                             i8 addrspace(1)* nocapture %dst) {
2613entry:
2614  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
2615  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
2616
2617  %cvec1 = zext <8 x i4> %vec1 to <8 x i8>
2618  %cvec2 = zext <8 x i4> %vec2 to <8 x i8>
2619
2620  %mul = mul <8 x i8> %cvec1, %cvec2
2621  %mul0 = extractelement <8 x i8> %mul, i64 0
2622  %mul1 = extractelement <8 x i8> %mul, i64 1
2623  %mul2 = extractelement <8 x i8> %mul, i64 2
2624  %mul3 = extractelement <8 x i8> %mul, i64 3
2625  %mul4 = extractelement <8 x i8> %mul, i64 4
2626  %mul5 = extractelement <8 x i8> %mul, i64 5
2627  %mul6 = extractelement <8 x i8> %mul, i64 6
2628  %mul7 = extractelement <8 x i8> %mul, i64 7
2629
2630  %acc = load i8, i8 addrspace(1)* %dst, align 4
2631  %add1 = add i8 %mul0, %acc
2632  %add2 = add i8 %add1, %mul1
2633  %add3 = add i8 %add2, %mul2
2634  %add4 = add i8 %add3, %mul3
2635  %add5 = add i8 %add4, %mul4
2636  %add6 = add i8 %add5, %mul5
2637  %add7 = add i8 %add6, %mul6
2638  %add8 = add i8 %add7, %mul7
2639
2640  store i8 %add8, i8 addrspace(1)* %dst, align 4
2641  ret void
2642}
2643
2644; TODO: Once the adictional "and+add" are removed, the pattern will be recognized.
2645define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
2646; GFX7-LABEL: udot8_acc4_vecMul:
2647; GFX7:       ; %bb.0: ; %entry
2648; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
2649; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
2650; GFX7-NEXT:    s_mov_b32 s7, 0xf000
2651; GFX7-NEXT:    s_mov_b32 s6, -1
2652; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2653; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
2654; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
2655; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
2656; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2657; GFX7-NEXT:    s_lshr_b32 s2, s0, 28
2658; GFX7-NEXT:    s_bfe_u32 s15, s1, 0x40018
2659; GFX7-NEXT:    s_bfe_u32 s16, s1, 0x40014
2660; GFX7-NEXT:    s_bfe_u32 s17, s1, 0x40010
2661; GFX7-NEXT:    s_bfe_u32 s18, s1, 0x4000c
2662; GFX7-NEXT:    s_bfe_u32 s19, s1, 0x40008
2663; GFX7-NEXT:    s_bfe_u32 s20, s1, 0x40004
2664; GFX7-NEXT:    s_lshr_b32 s14, s1, 28
2665; GFX7-NEXT:    s_and_b32 s1, s1, 15
2666; GFX7-NEXT:    s_bfe_u32 s8, s0, 0x40018
2667; GFX7-NEXT:    s_bfe_u32 s9, s0, 0x40014
2668; GFX7-NEXT:    s_bfe_u32 s10, s0, 0x40010
2669; GFX7-NEXT:    s_bfe_u32 s11, s0, 0x4000c
2670; GFX7-NEXT:    s_bfe_u32 s12, s0, 0x40008
2671; GFX7-NEXT:    s_bfe_u32 s13, s0, 0x40004
2672; GFX7-NEXT:    s_and_b32 s0, s0, 15
2673; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2674; GFX7-NEXT:    v_mov_b32_e32 v2, s20
2675; GFX7-NEXT:    v_mov_b32_e32 v3, s19
2676; GFX7-NEXT:    v_mov_b32_e32 v4, s18
2677; GFX7-NEXT:    v_mov_b32_e32 v5, s17
2678; GFX7-NEXT:    v_mov_b32_e32 v6, s16
2679; GFX7-NEXT:    v_mov_b32_e32 v7, s15
2680; GFX7-NEXT:    s_waitcnt vmcnt(0)
2681; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
2682; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v2, v0
2683; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
2684; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v4, v0
2685; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v5, v0
2686; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v6, v0
2687; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v7, v0
2688; GFX7-NEXT:    v_mov_b32_e32 v1, s14
2689; GFX7-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
2690; GFX7-NEXT:    v_and_b32_e32 v0, 15, v0
2691; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
2692; GFX7-NEXT:    s_endpgm
2693;
2694; GFX8-LABEL: udot8_acc4_vecMul:
2695; GFX8:       ; %bb.0: ; %entry
2696; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2697; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2698; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2699; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
2700; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2701; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2702; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
2703; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
2704; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2705; GFX8-NEXT:    s_and_b32 s1, s2, 15
2706; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x40004
2707; GFX8-NEXT:    v_mov_b32_e32 v3, s1
2708; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40008
2709; GFX8-NEXT:    s_and_b32 s1, s0, 15
2710; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x4000c
2711; GFX8-NEXT:    v_mov_b32_e32 v4, s4
2712; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x40004
2713; GFX8-NEXT:    v_mov_b32_e32 v5, s7
2714; GFX8-NEXT:    s_bfe_u32 s8, s0, 0x4000c
2715; GFX8-NEXT:    v_mov_b32_e32 v6, s6
2716; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x40008
2717; GFX8-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
2718; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40010
2719; GFX8-NEXT:    v_and_b32_e32 v5, 15, v5
2720; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40014
2721; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x40010
2722; GFX8-NEXT:    v_mov_b32_e32 v7, s6
2723; GFX8-NEXT:    s_lshr_b32 s11, s2, 28
2724; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x40018
2725; GFX8-NEXT:    s_bfe_u32 s9, s0, 0x40014
2726; GFX8-NEXT:    v_mov_b32_e32 v8, s8
2727; GFX8-NEXT:    s_bfe_u32 s10, s0, 0x40018
2728; GFX8-NEXT:    v_mov_b32_e32 v9, s2
2729; GFX8-NEXT:    s_lshr_b32 s0, s0, 28
2730; GFX8-NEXT:    s_waitcnt vmcnt(0)
2731; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
2732; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
2733; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
2734; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
2735; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v5, v2
2736; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
2737; GFX8-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
2738; GFX8-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
2739; GFX8-NEXT:    v_mov_b32_e32 v3, s11
2740; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
2741; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
2742; GFX8-NEXT:    flat_store_byte v[0:1], v2
2743; GFX8-NEXT:    s_endpgm
2744;
2745; GFX9-LABEL: udot8_acc4_vecMul:
2746; GFX9:       ; %bb.0: ; %entry
2747; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2748; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2749; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2750; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
2751; GFX9-NEXT:    v_mov_b32_e32 v0, s0
2752; GFX9-NEXT:    v_mov_b32_e32 v1, s1
2753; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
2754; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
2755; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2756; GFX9-NEXT:    s_and_b32 s1, s2, 15
2757; GFX9-NEXT:    s_bfe_u32 s4, s2, 0x40004
2758; GFX9-NEXT:    v_mov_b32_e32 v3, s1
2759; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40008
2760; GFX9-NEXT:    s_and_b32 s1, s0, 15
2761; GFX9-NEXT:    s_bfe_u32 s7, s2, 0x4000c
2762; GFX9-NEXT:    v_mov_b32_e32 v4, s4
2763; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x40004
2764; GFX9-NEXT:    v_mov_b32_e32 v5, s7
2765; GFX9-NEXT:    s_bfe_u32 s8, s0, 0x4000c
2766; GFX9-NEXT:    v_mov_b32_e32 v6, s6
2767; GFX9-NEXT:    s_bfe_u32 s4, s0, 0x40008
2768; GFX9-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
2769; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40010
2770; GFX9-NEXT:    v_and_b32_e32 v5, 15, v5
2771; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
2772; GFX9-NEXT:    s_bfe_u32 s7, s0, 0x40010
2773; GFX9-NEXT:    v_mov_b32_e32 v7, s6
2774; GFX9-NEXT:    s_lshr_b32 s11, s2, 28
2775; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x40018
2776; GFX9-NEXT:    s_bfe_u32 s9, s0, 0x40014
2777; GFX9-NEXT:    v_mov_b32_e32 v8, s8
2778; GFX9-NEXT:    s_bfe_u32 s10, s0, 0x40018
2779; GFX9-NEXT:    v_mov_b32_e32 v9, s2
2780; GFX9-NEXT:    s_lshr_b32 s0, s0, 28
2781; GFX9-NEXT:    s_waitcnt vmcnt(0)
2782; GFX9-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
2783; GFX9-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
2784; GFX9-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
2785; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
2786; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
2787; GFX9-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
2788; GFX9-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
2789; GFX9-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
2790; GFX9-NEXT:    v_mov_b32_e32 v3, s11
2791; GFX9-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
2792; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
2793; GFX9-NEXT:    global_store_byte v[0:1], v2, off
2794; GFX9-NEXT:    s_endpgm
2795;
2796; GFX9-DL-LABEL: udot8_acc4_vecMul:
2797; GFX9-DL:       ; %bb.0: ; %entry
2798; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2799; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2800; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2801; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
2802; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2803; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2804; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
2805; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
2806; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2807; GFX9-DL-NEXT:    s_and_b32 s1, s2, 15
2808; GFX9-DL-NEXT:    s_bfe_u32 s4, s2, 0x40004
2809; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
2810; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40008
2811; GFX9-DL-NEXT:    s_and_b32 s1, s0, 15
2812; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x4000c
2813; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
2814; GFX9-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
2815; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s7
2816; GFX9-DL-NEXT:    s_bfe_u32 s8, s0, 0x4000c
2817; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s6
2818; GFX9-DL-NEXT:    s_bfe_u32 s4, s0, 0x40008
2819; GFX9-DL-NEXT:    v_mul_u32_u24_e32 v5, s8, v5
2820; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x40010
2821; GFX9-DL-NEXT:    v_and_b32_e32 v5, 15, v5
2822; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40014
2823; GFX9-DL-NEXT:    s_bfe_u32 s7, s0, 0x40010
2824; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s6
2825; GFX9-DL-NEXT:    s_lshr_b32 s11, s2, 28
2826; GFX9-DL-NEXT:    s_bfe_u32 s2, s2, 0x40018
2827; GFX9-DL-NEXT:    s_bfe_u32 s9, s0, 0x40014
2828; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s8
2829; GFX9-DL-NEXT:    s_bfe_u32 s10, s0, 0x40018
2830; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s2
2831; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 28
2832; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2833; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
2834; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
2835; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v6, v2
2836; GFX9-DL-NEXT:    v_and_b32_e32 v2, 15, v2
2837; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
2838; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v7, v2
2839; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s9, v8, v2
2840; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s10, v9, v2
2841; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s11
2842; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
2843; GFX9-DL-NEXT:    v_and_b32_e32 v2, 15, v2
2844; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
2845; GFX9-DL-NEXT:    s_endpgm
2846;
2847; GFX10-DL-LABEL: udot8_acc4_vecMul:
2848; GFX10-DL:       ; %bb.0: ; %entry
2849; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2850; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2851; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2852; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2853; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2854; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2855; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
2856; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
2857; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
2858; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2859; GFX10-DL-NEXT:    s_and_b32 s2, s0, 15
2860; GFX10-DL-NEXT:    s_and_b32 s4, s1, 15
2861; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40004
2862; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40004
2863; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x40008
2864; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
2865; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
2866; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40008
2867; GFX10-DL-NEXT:    s_bfe_u32 s4, s0, 0x4000c
2868; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
2869; GFX10-DL-NEXT:    s_bfe_u32 s5, s1, 0x4000c
2870; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40014
2871; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s7, v2
2872; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v3, s4, s5
2873; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40010
2874; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40010
2875; GFX10-DL-NEXT:    s_bfe_u32 s5, s0, 0x40014
2876; GFX10-DL-NEXT:    v_and_b32_e32 v2, 15, v2
2877; GFX10-DL-NEXT:    v_and_b32_e32 v3, 15, v3
2878; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
2879; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
2880; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
2881; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x40018
2882; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 28
2883; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
2884; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s5, s6, v2
2885; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s4, v2
2886; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
2887; GFX10-DL-NEXT:    v_and_b32_e32 v2, 15, v2
2888; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
2889; GFX10-DL-NEXT:    s_endpgm
2890                                             <8 x i4> addrspace(1)* %src2,
2891                                             i4 addrspace(1)* nocapture %dst) {
2892entry:
2893  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
2894  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
2895
2896  %mul = mul <8 x i4> %vec1, %vec2
2897  %mul0 = extractelement <8 x i4> %mul, i64 0
2898  %mul1 = extractelement <8 x i4> %mul, i64 1
2899  %mul2 = extractelement <8 x i4> %mul, i64 2
2900  %mul3 = extractelement <8 x i4> %mul, i64 3
2901  %mul4 = extractelement <8 x i4> %mul, i64 4
2902  %mul5 = extractelement <8 x i4> %mul, i64 5
2903  %mul6 = extractelement <8 x i4> %mul, i64 6
2904  %mul7 = extractelement <8 x i4> %mul, i64 7
2905
2906  %acc = load i4, i4 addrspace(1)* %dst, align 4
2907  %add1 = add i4 %mul0, %acc
2908  %add2 = add i4 %add1, %mul1
2909  %add3 = add i4 %add2, %mul2
2910  %add4 = add i4 %add3, %mul3
2911  %add5 = add i4 %add4, %mul4
2912  %add6 = add i4 %add5, %mul5
2913  %add7 = add i4 %add6, %mul6
2914  %add8 = add i4 %add7, %mul7
2915
2916  store i4 %add8, i4 addrspace(1)* %dst, align 4
2917  ret void
2918}
2919
2920define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
2921; GFX7-LABEL: udot8_variant1:
2922; GFX7:       ; %bb.0: ; %entry
2923; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2924; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2925; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2926; GFX7-NEXT:    s_mov_b32 s2, -1
2927; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2928; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2929; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2930; GFX7-NEXT:    s_load_dword s20, s[0:1], 0x0
2931; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2932; GFX7-NEXT:    s_and_b32 s6, s4, 15
2933; GFX7-NEXT:    s_and_b32 s7, s5, 15
2934; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x40004
2935; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x40008
2936; GFX7-NEXT:    s_bfe_u32 s12, s4, 0x4000c
2937; GFX7-NEXT:    s_bfe_u32 s14, s4, 0x40010
2938; GFX7-NEXT:    s_bfe_u32 s16, s4, 0x40014
2939; GFX7-NEXT:    s_bfe_u32 s18, s4, 0x40018
2940; GFX7-NEXT:    s_lshr_b32 s4, s4, 28
2941; GFX7-NEXT:    v_mov_b32_e32 v0, s6
2942; GFX7-NEXT:    v_mov_b32_e32 v1, s20
2943; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
2944; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x40004
2945; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x40008
2946; GFX7-NEXT:    s_bfe_u32 s13, s5, 0x4000c
2947; GFX7-NEXT:    s_bfe_u32 s15, s5, 0x40010
2948; GFX7-NEXT:    s_bfe_u32 s17, s5, 0x40014
2949; GFX7-NEXT:    s_bfe_u32 s19, s5, 0x40018
2950; GFX7-NEXT:    s_lshr_b32 s5, s5, 28
2951; GFX7-NEXT:    v_mov_b32_e32 v1, s4
2952; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
2953; GFX7-NEXT:    v_mov_b32_e32 v1, s8
2954; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
2955; GFX7-NEXT:    v_mov_b32_e32 v1, s10
2956; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
2957; GFX7-NEXT:    v_mov_b32_e32 v1, s12
2958; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v1, v0
2959; GFX7-NEXT:    v_mov_b32_e32 v1, s14
2960; GFX7-NEXT:    v_mad_u32_u24 v0, s15, v1, v0
2961; GFX7-NEXT:    v_mov_b32_e32 v1, s16
2962; GFX7-NEXT:    v_mad_u32_u24 v0, s17, v1, v0
2963; GFX7-NEXT:    v_mov_b32_e32 v1, s18
2964; GFX7-NEXT:    v_mad_u32_u24 v0, s19, v1, v0
2965; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2966; GFX7-NEXT:    s_endpgm
2967;
2968; GFX8-LABEL: udot8_variant1:
2969; GFX8:       ; %bb.0: ; %entry
2970; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2971; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2972; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2973; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
2974; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
2975; GFX8-NEXT:    s_load_dword s18, s[0:1], 0x0
2976; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2977; GFX8-NEXT:    s_and_b32 s4, s2, 15
2978; GFX8-NEXT:    s_and_b32 s5, s3, 15
2979; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x40004
2980; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x40008
2981; GFX8-NEXT:    s_bfe_u32 s10, s2, 0x4000c
2982; GFX8-NEXT:    s_bfe_u32 s12, s2, 0x40010
2983; GFX8-NEXT:    s_bfe_u32 s14, s2, 0x40014
2984; GFX8-NEXT:    s_bfe_u32 s16, s2, 0x40018
2985; GFX8-NEXT:    s_lshr_b32 s2, s2, 28
2986; GFX8-NEXT:    v_mov_b32_e32 v0, s4
2987; GFX8-NEXT:    v_mov_b32_e32 v1, s18
2988; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
2989; GFX8-NEXT:    s_bfe_u32 s7, s3, 0x40004
2990; GFX8-NEXT:    s_bfe_u32 s9, s3, 0x40008
2991; GFX8-NEXT:    s_bfe_u32 s11, s3, 0x4000c
2992; GFX8-NEXT:    s_bfe_u32 s13, s3, 0x40010
2993; GFX8-NEXT:    s_bfe_u32 s15, s3, 0x40014
2994; GFX8-NEXT:    s_bfe_u32 s17, s3, 0x40018
2995; GFX8-NEXT:    s_lshr_b32 s3, s3, 28
2996; GFX8-NEXT:    v_mov_b32_e32 v1, s2
2997; GFX8-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
2998; GFX8-NEXT:    v_mov_b32_e32 v1, s6
2999; GFX8-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
3000; GFX8-NEXT:    v_mov_b32_e32 v1, s8
3001; GFX8-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
3002; GFX8-NEXT:    v_mov_b32_e32 v1, s10
3003; GFX8-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
3004; GFX8-NEXT:    v_mov_b32_e32 v1, s12
3005; GFX8-NEXT:    v_mad_u32_u24 v0, s13, v1, v0
3006; GFX8-NEXT:    v_mov_b32_e32 v1, s14
3007; GFX8-NEXT:    v_mad_u32_u24 v0, s15, v1, v0
3008; GFX8-NEXT:    v_mov_b32_e32 v1, s16
3009; GFX8-NEXT:    v_mad_u32_u24 v2, s17, v1, v0
3010; GFX8-NEXT:    v_mov_b32_e32 v0, s0
3011; GFX8-NEXT:    v_mov_b32_e32 v1, s1
3012; GFX8-NEXT:    flat_store_dword v[0:1], v2
3013; GFX8-NEXT:    s_endpgm
3014;
3015; GFX9-LABEL: udot8_variant1:
3016; GFX9:       ; %bb.0: ; %entry
3017; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
3018; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
3019; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
3020; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
3021; GFX9-NEXT:    s_load_dword s3, s[6:7], 0x0
3022; GFX9-NEXT:    s_load_dword s18, s[0:1], 0x0
3023; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
3024; GFX9-NEXT:    s_and_b32 s4, s2, 15
3025; GFX9-NEXT:    s_and_b32 s5, s3, 15
3026; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x40004
3027; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40008
3028; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x4000c
3029; GFX9-NEXT:    s_bfe_u32 s12, s2, 0x40010
3030; GFX9-NEXT:    s_bfe_u32 s14, s2, 0x40014
3031; GFX9-NEXT:    s_bfe_u32 s16, s2, 0x40018
3032; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
3033; GFX9-NEXT:    v_mov_b32_e32 v0, s4
3034; GFX9-NEXT:    v_mov_b32_e32 v1, s18
3035; GFX9-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
3036; GFX9-NEXT:    s_bfe_u32 s7, s3, 0x40004
3037; GFX9-NEXT:    s_bfe_u32 s9, s3, 0x40008
3038; GFX9-NEXT:    s_bfe_u32 s11, s3, 0x4000c
3039; GFX9-NEXT:    s_bfe_u32 s13, s3, 0x40010
3040; GFX9-NEXT:    s_bfe_u32 s15, s3, 0x40014
3041; GFX9-NEXT:    s_bfe_u32 s17, s3, 0x40018
3042; GFX9-NEXT:    s_lshr_b32 s3, s3, 28
3043; GFX9-NEXT:    v_mov_b32_e32 v1, s2
3044; GFX9-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
3045; GFX9-NEXT:    v_mov_b32_e32 v1, s6
3046; GFX9-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
3047; GFX9-NEXT:    v_mov_b32_e32 v1, s8
3048; GFX9-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
3049; GFX9-NEXT:    v_mov_b32_e32 v1, s10
3050; GFX9-NEXT:    v_mad_u32_u24 v0, s11, v1, v0
3051; GFX9-NEXT:    v_mov_b32_e32 v1, s12
3052; GFX9-NEXT:    v_mad_u32_u24 v0, s13, v1, v0
3053; GFX9-NEXT:    v_mov_b32_e32 v1, s14
3054; GFX9-NEXT:    v_mad_u32_u24 v0, s15, v1, v0
3055; GFX9-NEXT:    v_mov_b32_e32 v1, s16
3056; GFX9-NEXT:    v_mad_u32_u24 v2, s17, v1, v0
3057; GFX9-NEXT:    v_mov_b32_e32 v0, s0
3058; GFX9-NEXT:    v_mov_b32_e32 v1, s1
3059; GFX9-NEXT:    global_store_dword v[0:1], v2, off
3060; GFX9-NEXT:    s_endpgm
3061;
3062; GFX9-DL-LABEL: udot8_variant1:
3063; GFX9-DL:       ; %bb.0: ; %entry
3064; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
3065; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
3066; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
3067; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
3068; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
3069; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
3070; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
3071; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
3072; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
3073; GFX9-DL-NEXT:    v_dot8_u32_u4 v2, s4, v0, v1
3074; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
3075; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
3076; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
3077; GFX9-DL-NEXT:    s_endpgm
3078;
3079; GFX10-DL-LABEL: udot8_variant1:
3080; GFX10-DL:       ; %bb.0: ; %entry
3081; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
3082; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
3083; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
3084; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
3085; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
3086; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
3087; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
3088; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
3089; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
3090; GFX10-DL-NEXT:    v_dot8_u32_u4 v2, s4, s3, v0
3091; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
3092; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
3093; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
3094; GFX10-DL-NEXT:    s_endpgm
3095                                          i32 addrspace(1)* %v2addr,
3096                                          i32 addrspace(1)* %dst) {
3097entry:
3098  %v1 = load i32, i32 addrspace(1)* %v1addr, align 4
3099  %v2 = load i32, i32 addrspace(1)* %v2addr, align 4
3100  %and = and i32 %v1, 15
3101  %and1 = and i32 %v2, 15
3102  %mul1 = mul nuw nsw i32 %and1, %and
3103
3104  %shr = lshr i32 %v1, 4
3105  %and2 = and i32 %shr, 15
3106  %shr3 = lshr i32 %v2, 4
3107  %and4 = and i32 %shr3, 15
3108  %mul2 = mul nuw nsw i32 %and4, %and2
3109
3110  %shr6 = lshr i32 %v1, 8
3111  %and7 = and i32 %shr6, 15
3112  %shr8 = lshr i32 %v2, 8
3113  %and9 = and i32 %shr8, 15
3114  %mul3 = mul nuw nsw i32 %and9, %and7
3115
3116  %shr12 = lshr i32 %v1, 12
3117  %and13 = and i32 %shr12, 15
3118  %shr14 = lshr i32 %v2, 12
3119  %and15 = and i32 %shr14, 15
3120  %mul4 = mul nuw nsw i32 %and15, %and13
3121
3122  %shr18 = lshr i32 %v1, 16
3123  %and19 = and i32 %shr18, 15
3124  %shr20 = lshr i32 %v2, 16
3125  %and21 = and i32 %shr20, 15
3126  %mul5 = mul nuw nsw i32 %and21, %and19
3127
3128  %shr24 = lshr i32 %v1, 20
3129  %and25 = and i32 %shr24, 15
3130  %shr26 = lshr i32 %v2, 20
3131  %and27 = and i32 %shr26, 15
3132  %mul6 = mul nuw nsw i32 %and27, %and25
3133
3134  %shr30 = lshr i32 %v1, 24
3135  %and31 = and i32 %shr30, 15
3136  %shr32 = lshr i32 %v2, 24
3137  %and33 = and i32 %shr32, 15
3138  %mul7 = mul nuw nsw i32 %and33, %and31
3139
3140  %shr36 = lshr i32 %v1, 28
3141  %shr37 = lshr i32 %v2, 28
3142  %mul8 = mul nuw nsw i32 %shr37, %shr36
3143  %acc = load i32, i32 addrspace(1)* %dst, align 4
3144
3145  %add1 = add i32 %mul1, %acc
3146  %add2 = add i32 %add1, %mul8
3147  %add3 = add i32 %add2, %mul2
3148  %add4 = add i32 %add3, %mul3
3149  %add5 = add i32 %add4, %mul4
3150  %add6 = add i32 %add5, %mul5
3151  %add7 = add i32 %add6, %mul6
3152  %add8 = add i32 %add7, %mul7
3153  store i32 %add8, i32 addrspace(1)* %dst, align 4
3154  ret void
3155}
3156