1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
8
9define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
10; GFX7-LABEL: idot8_acc32:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
13; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
14; GFX7-NEXT:    s_mov_b32 s7, 0xf000
15; GFX7-NEXT:    s_mov_b32 s6, -1
16; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
17; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
18; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
19; GFX7-NEXT:    s_load_dword s21, s[4:5], 0x0
20; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX7-NEXT:    s_bfe_i32 s2, s0, 0x40000
22; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40000
23; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40004
24; GFX7-NEXT:    v_mov_b32_e32 v0, s8
25; GFX7-NEXT:    v_mov_b32_e32 v1, s21
26; GFX7-NEXT:    v_mad_i32_i24 v0, s2, v0, v1
27; GFX7-NEXT:    s_bfe_i32 s9, s0, 0x40004
28; GFX7-NEXT:    v_mov_b32_e32 v1, s10
29; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40008
30; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
31; GFX7-NEXT:    s_bfe_i32 s11, s0, 0x40008
32; GFX7-NEXT:    v_mov_b32_e32 v1, s12
33; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x4000c
34; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
35; GFX7-NEXT:    s_bfe_i32 s13, s0, 0x4000c
36; GFX7-NEXT:    v_mov_b32_e32 v1, s14
37; GFX7-NEXT:    s_bfe_i32 s16, s1, 0x40010
38; GFX7-NEXT:    v_mad_i32_i24 v0, s13, v1, v0
39; GFX7-NEXT:    s_bfe_i32 s15, s0, 0x40010
40; GFX7-NEXT:    v_mov_b32_e32 v1, s16
41; GFX7-NEXT:    s_bfe_i32 s18, s1, 0x40014
42; GFX7-NEXT:    s_bfe_i32 s20, s1, 0x40018
43; GFX7-NEXT:    v_mad_i32_i24 v0, s15, v1, v0
44; GFX7-NEXT:    s_bfe_i32 s17, s0, 0x40014
45; GFX7-NEXT:    v_mov_b32_e32 v1, s18
46; GFX7-NEXT:    s_bfe_i32 s19, s0, 0x40018
47; GFX7-NEXT:    v_mad_i32_i24 v0, s17, v1, v0
48; GFX7-NEXT:    v_mov_b32_e32 v1, s20
49; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
50; GFX7-NEXT:    v_mad_i32_i24 v0, s19, v1, v0
51; GFX7-NEXT:    s_ashr_i32 s0, s0, 28
52; GFX7-NEXT:    v_mov_b32_e32 v1, s1
53; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v1, v0
54; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
55; GFX7-NEXT:    s_endpgm
56;
57; GFX8-LABEL: idot8_acc32:
58; GFX8:       ; %bb.0: ; %entry
59; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
60; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
61; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
62; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
63; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
64; GFX8-NEXT:    s_load_dword s19, s[0:1], 0x0
65; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
66; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x40000
67; GFX8-NEXT:    s_bfe_i32 s6, s4, 0x40000
68; GFX8-NEXT:    s_bfe_i32 s8, s4, 0x40004
69; GFX8-NEXT:    v_mov_b32_e32 v0, s6
70; GFX8-NEXT:    v_mov_b32_e32 v1, s19
71; GFX8-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
72; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x40004
73; GFX8-NEXT:    v_mov_b32_e32 v1, s8
74; GFX8-NEXT:    s_bfe_i32 s10, s4, 0x40008
75; GFX8-NEXT:    v_mad_i32_i24 v0, s7, v1, v0
76; GFX8-NEXT:    s_bfe_i32 s9, s2, 0x40008
77; GFX8-NEXT:    v_mov_b32_e32 v1, s10
78; GFX8-NEXT:    s_bfe_i32 s12, s4, 0x4000c
79; GFX8-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
80; GFX8-NEXT:    s_bfe_i32 s11, s2, 0x4000c
81; GFX8-NEXT:    v_mov_b32_e32 v1, s12
82; GFX8-NEXT:    s_bfe_i32 s14, s4, 0x40010
83; GFX8-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
84; GFX8-NEXT:    s_bfe_i32 s13, s2, 0x40010
85; GFX8-NEXT:    v_mov_b32_e32 v1, s14
86; GFX8-NEXT:    s_bfe_i32 s16, s4, 0x40014
87; GFX8-NEXT:    s_bfe_i32 s18, s4, 0x40018
88; GFX8-NEXT:    v_mad_i32_i24 v0, s13, v1, v0
89; GFX8-NEXT:    s_bfe_i32 s15, s2, 0x40014
90; GFX8-NEXT:    v_mov_b32_e32 v1, s16
91; GFX8-NEXT:    s_bfe_i32 s17, s2, 0x40018
92; GFX8-NEXT:    v_mad_i32_i24 v0, s15, v1, v0
93; GFX8-NEXT:    v_mov_b32_e32 v1, s18
94; GFX8-NEXT:    s_ashr_i32 s4, s4, 28
95; GFX8-NEXT:    v_mad_i32_i24 v0, s17, v1, v0
96; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
97; GFX8-NEXT:    v_mov_b32_e32 v1, s4
98; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
99; GFX8-NEXT:    v_mov_b32_e32 v0, s0
100; GFX8-NEXT:    v_mov_b32_e32 v1, s1
101; GFX8-NEXT:    flat_store_dword v[0:1], v2
102; GFX8-NEXT:    s_endpgm
103;
104; GFX9-LABEL: idot8_acc32:
105; GFX9:       ; %bb.0: ; %entry
106; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
107; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
108; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
109; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
110; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
111; GFX9-NEXT:    s_load_dword s19, s[0:1], 0x0
112; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
113; GFX9-NEXT:    s_bfe_i32 s5, s2, 0x40000
114; GFX9-NEXT:    s_bfe_i32 s6, s4, 0x40000
115; GFX9-NEXT:    s_bfe_i32 s8, s4, 0x40004
116; GFX9-NEXT:    v_mov_b32_e32 v0, s6
117; GFX9-NEXT:    v_mov_b32_e32 v1, s19
118; GFX9-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
119; GFX9-NEXT:    s_bfe_i32 s7, s2, 0x40004
120; GFX9-NEXT:    v_mov_b32_e32 v1, s8
121; GFX9-NEXT:    s_bfe_i32 s10, s4, 0x40008
122; GFX9-NEXT:    v_mad_i32_i24 v0, s7, v1, v0
123; GFX9-NEXT:    s_bfe_i32 s9, s2, 0x40008
124; GFX9-NEXT:    v_mov_b32_e32 v1, s10
125; GFX9-NEXT:    s_bfe_i32 s12, s4, 0x4000c
126; GFX9-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
127; GFX9-NEXT:    s_bfe_i32 s11, s2, 0x4000c
128; GFX9-NEXT:    v_mov_b32_e32 v1, s12
129; GFX9-NEXT:    s_bfe_i32 s14, s4, 0x40010
130; GFX9-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
131; GFX9-NEXT:    s_bfe_i32 s13, s2, 0x40010
132; GFX9-NEXT:    v_mov_b32_e32 v1, s14
133; GFX9-NEXT:    s_bfe_i32 s16, s4, 0x40014
134; GFX9-NEXT:    s_bfe_i32 s18, s4, 0x40018
135; GFX9-NEXT:    v_mad_i32_i24 v0, s13, v1, v0
136; GFX9-NEXT:    s_bfe_i32 s15, s2, 0x40014
137; GFX9-NEXT:    v_mov_b32_e32 v1, s16
138; GFX9-NEXT:    s_bfe_i32 s17, s2, 0x40018
139; GFX9-NEXT:    v_mad_i32_i24 v0, s15, v1, v0
140; GFX9-NEXT:    v_mov_b32_e32 v1, s18
141; GFX9-NEXT:    s_ashr_i32 s4, s4, 28
142; GFX9-NEXT:    v_mad_i32_i24 v0, s17, v1, v0
143; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
144; GFX9-NEXT:    v_mov_b32_e32 v1, s4
145; GFX9-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
146; GFX9-NEXT:    v_mov_b32_e32 v0, s0
147; GFX9-NEXT:    v_mov_b32_e32 v1, s1
148; GFX9-NEXT:    global_store_dword v[0:1], v2, off
149; GFX9-NEXT:    s_endpgm
150;
151; GFX9-DL-LABEL: idot8_acc32:
152; GFX9-DL:       ; %bb.0: ; %entry
153; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
154; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
155; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
156; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
157; GFX9-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
158; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
159; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
160; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
161; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
162; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
163; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s6
164; GFX9-DL-NEXT:    v_dot8_i32_i4 v2, s4, v2, v3
165; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
166; GFX9-DL-NEXT:    s_endpgm
167;
168; GFX10-DL-LABEL: idot8_acc32:
169; GFX10-DL:       ; %bb.0: ; %entry
170; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
171; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
172; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
173; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
174; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
175; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
176; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
177; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
178; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
179; GFX10-DL-NEXT:    v_dot8_i32_i4 v2, s4, s5, v0
180; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
181; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
182; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
183; GFX10-DL-NEXT:    s_endpgm
184                                       <8 x i4> addrspace(1)* %src2,
185                                       i32 addrspace(1)* nocapture %dst) {
186entry:
187  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
188  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
189
190  %v1e0 = extractelement <8 x i4> %vec1, i64 0
191  %cv1e0 = sext i4 %v1e0 to i32
192  %v2e0 = extractelement <8 x i4> %vec2, i64 0
193  %cv2e0 = sext i4 %v2e0 to i32
194  %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
195
196  %v1e1 = extractelement <8 x i4> %vec1, i64 1
197  %cv1e1 = sext i4 %v1e1 to i32
198  %v2e1 = extractelement <8 x i4> %vec2, i64 1
199  %cv2e1 = sext i4 %v2e1 to i32
200  %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
201
202  %v1e2 = extractelement <8 x i4> %vec1, i64 2
203  %cv1e2 = sext i4 %v1e2 to i32
204  %v2e2 = extractelement <8 x i4> %vec2, i64 2
205  %cv2e2 = sext i4 %v2e2 to i32
206  %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
207
208  %v1e3 = extractelement <8 x i4> %vec1, i64 3
209  %cv1e3 = sext i4 %v1e3 to i32
210  %v2e3 = extractelement <8 x i4> %vec2, i64 3
211  %cv2e3 = sext i4 %v2e3 to i32
212  %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
213
214  %v1e4 = extractelement <8 x i4> %vec1, i64 4
215  %cv1e4 = sext i4 %v1e4 to i32
216  %v2e4 = extractelement <8 x i4> %vec2, i64 4
217  %cv2e4 = sext i4 %v2e4 to i32
218  %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
219
220  %v1e5 = extractelement <8 x i4> %vec1, i64 5
221  %cv1e5 = sext i4 %v1e5 to i32
222  %v2e5 = extractelement <8 x i4> %vec2, i64 5
223  %cv2e5 = sext i4 %v2e5 to i32
224  %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
225
226  %v1e6 = extractelement <8 x i4> %vec1, i64 6
227  %cv1e6 = sext i4 %v1e6 to i32
228  %v2e6 = extractelement <8 x i4> %vec2, i64 6
229  %cv2e6 = sext i4 %v2e6 to i32
230  %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
231
232  %v1e7 = extractelement <8 x i4> %vec1, i64 7
233  %cv1e7 = sext i4 %v1e7 to i32
234  %v2e7 = extractelement <8 x i4> %vec2, i64 7
235  %cv2e7 = sext i4 %v2e7 to i32
236  %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
237
238  %acc = load i32, i32 addrspace(1)* %dst, align 4
239  %add1 = add i32 %mul0, %acc
240  %add2 = add i32 %add1, %mul1
241  %add3 = add i32 %add2, %mul2
242  %add4 = add i32 %add3, %mul3
243  %add5 = add i32 %add4, %mul4
244  %add6 = add i32 %add5, %mul5
245  %add7 = add i32 %add6, %mul6
246  %add8 = add i32 %add7, %mul7
247
248  store i32 %add8, i32 addrspace(1)* %dst, align 4
249  ret void
250}
251
252; TODO: Once the unnecessary zero extentions of the elements are removed;
253; pattern recognizer will kick in.
254define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
255; GFX7-LABEL: idot8_acc16:
256; GFX7:       ; %bb.0: ; %entry
257; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
258; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
259; GFX7-NEXT:    s_mov_b32 s7, 0xf000
260; GFX7-NEXT:    s_mov_b32 s6, -1
261; GFX7-NEXT:    s_mov_b32 s0, 0xffff
262; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
263; GFX7-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
264; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
265; GFX7-NEXT:    s_load_dword s2, s[10:11], 0x0
266; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
267; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40000
268; GFX7-NEXT:    s_bfe_i32 s9, s2, 0x40000
269; GFX7-NEXT:    s_bfe_i32 s11, s2, 0x40004
270; GFX7-NEXT:    s_and_b32 s9, s9, s0
271; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40004
272; GFX7-NEXT:    s_bfe_i32 s13, s2, 0x40008
273; GFX7-NEXT:    s_and_b32 s11, s11, s0
274; GFX7-NEXT:    s_and_b32 s8, s8, s0
275; GFX7-NEXT:    v_mov_b32_e32 v1, s9
276; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40008
277; GFX7-NEXT:    s_bfe_i32 s15, s2, 0x4000c
278; GFX7-NEXT:    s_and_b32 s13, s13, s0
279; GFX7-NEXT:    s_and_b32 s10, s10, s0
280; GFX7-NEXT:    v_mov_b32_e32 v2, s11
281; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x4000c
282; GFX7-NEXT:    s_bfe_i32 s17, s2, 0x40010
283; GFX7-NEXT:    s_and_b32 s15, s15, s0
284; GFX7-NEXT:    s_and_b32 s12, s12, s0
285; GFX7-NEXT:    v_mov_b32_e32 v3, s13
286; GFX7-NEXT:    s_bfe_i32 s16, s1, 0x40010
287; GFX7-NEXT:    s_bfe_i32 s19, s2, 0x40014
288; GFX7-NEXT:    s_and_b32 s17, s17, s0
289; GFX7-NEXT:    s_and_b32 s14, s14, s0
290; GFX7-NEXT:    v_mov_b32_e32 v4, s15
291; GFX7-NEXT:    s_bfe_i32 s21, s2, 0x40018
292; GFX7-NEXT:    s_bfe_i32 s18, s1, 0x40014
293; GFX7-NEXT:    s_and_b32 s19, s19, s0
294; GFX7-NEXT:    s_and_b32 s16, s16, s0
295; GFX7-NEXT:    v_mov_b32_e32 v5, s17
296; GFX7-NEXT:    s_bfe_i32 s20, s1, 0x40018
297; GFX7-NEXT:    s_ashr_i32 s2, s2, 28
298; GFX7-NEXT:    s_and_b32 s21, s21, s0
299; GFX7-NEXT:    s_and_b32 s18, s18, s0
300; GFX7-NEXT:    v_mov_b32_e32 v6, s19
301; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
302; GFX7-NEXT:    s_and_b32 s20, s20, s0
303; GFX7-NEXT:    s_and_b32 s2, s2, s0
304; GFX7-NEXT:    v_mov_b32_e32 v7, s21
305; GFX7-NEXT:    s_and_b32 s0, s1, s0
306; GFX7-NEXT:    s_waitcnt vmcnt(0)
307; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
308; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v2, v0
309; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
310; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v4, v0
311; GFX7-NEXT:    v_mad_u32_u24 v0, s16, v5, v0
312; GFX7-NEXT:    v_mad_u32_u24 v0, s18, v6, v0
313; GFX7-NEXT:    v_mad_u32_u24 v0, s20, v7, v0
314; GFX7-NEXT:    v_mov_b32_e32 v1, s2
315; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
316; GFX7-NEXT:    buffer_store_short v0, off, s[4:7], 0
317; GFX7-NEXT:    s_endpgm
318;
319; GFX8-LABEL: idot8_acc16:
320; GFX8:       ; %bb.0: ; %entry
321; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
322; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
323; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
324; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
325; GFX8-NEXT:    v_mov_b32_e32 v0, s0
326; GFX8-NEXT:    v_mov_b32_e32 v1, s1
327; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
328; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
329; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
330; GFX8-NEXT:    s_bfe_i32 s1, s2, 0x40000
331; GFX8-NEXT:    s_bfe_i32 s4, s2, 0x40004
332; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x40008
333; GFX8-NEXT:    v_mov_b32_e32 v3, s1
334; GFX8-NEXT:    s_bfe_i32 s6, s0, 0x40000
335; GFX8-NEXT:    s_lshr_b32 s1, s0, 12
336; GFX8-NEXT:    s_lshr_b32 s7, s2, 12
337; GFX8-NEXT:    v_mov_b32_e32 v4, s5
338; GFX8-NEXT:    s_bfe_i32 s9, s0, 0x40008
339; GFX8-NEXT:    v_mov_b32_e32 v5, s4
340; GFX8-NEXT:    s_bfe_i32 s8, s0, 0x40004
341; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s1
342; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s7
343; GFX8-NEXT:    v_mul_i32_i24_e32 v4, s9, v4
344; GFX8-NEXT:    s_bfe_i32 s1, s2, 0x40010
345; GFX8-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
346; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
347; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x40014
348; GFX8-NEXT:    v_mov_b32_e32 v8, s1
349; GFX8-NEXT:    s_bfe_i32 s4, s0, 0x40010
350; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x40018
351; GFX8-NEXT:    v_mov_b32_e32 v9, s5
352; GFX8-NEXT:    s_bfe_i32 s1, s0, 0x40014
353; GFX8-NEXT:    s_bfe_i32 s5, s0, 0x40018
354; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
355; GFX8-NEXT:    v_mov_b32_e32 v10, s7
356; GFX8-NEXT:    s_ashr_i32 s0, s0, 28
357; GFX8-NEXT:    s_waitcnt vmcnt(0)
358; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v3, v2
359; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v5, v2
360; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
361; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v7, v2
362; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v8, v2
363; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v9, v2
364; GFX8-NEXT:    v_mad_i32_i24 v2, s5, v10, v2
365; GFX8-NEXT:    v_mov_b32_e32 v3, s2
366; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
367; GFX8-NEXT:    flat_store_short v[0:1], v2
368; GFX8-NEXT:    s_endpgm
369;
370; GFX9-LABEL: idot8_acc16:
371; GFX9:       ; %bb.0: ; %entry
372; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
373; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
374; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
375; GFX9-NEXT:    s_load_dword s2, s[6:7], 0x0
376; GFX9-NEXT:    v_mov_b32_e32 v0, s0
377; GFX9-NEXT:    v_mov_b32_e32 v1, s1
378; GFX9-NEXT:    global_load_ushort v2, v[0:1], off
379; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
380; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
381; GFX9-NEXT:    s_bfe_i32 s1, s2, 0x40000
382; GFX9-NEXT:    s_bfe_i32 s4, s2, 0x40004
383; GFX9-NEXT:    s_bfe_i32 s5, s2, 0x40008
384; GFX9-NEXT:    v_mov_b32_e32 v3, s1
385; GFX9-NEXT:    s_bfe_i32 s6, s0, 0x40000
386; GFX9-NEXT:    s_lshr_b32 s1, s0, 12
387; GFX9-NEXT:    s_lshr_b32 s7, s2, 12
388; GFX9-NEXT:    v_mov_b32_e32 v4, s5
389; GFX9-NEXT:    s_bfe_i32 s9, s0, 0x40008
390; GFX9-NEXT:    v_mov_b32_e32 v5, s4
391; GFX9-NEXT:    s_bfe_i32 s8, s0, 0x40004
392; GFX9-NEXT:    v_lshlrev_b16_e64 v6, 12, s1
393; GFX9-NEXT:    v_lshlrev_b16_e64 v7, 12, s7
394; GFX9-NEXT:    v_mul_i32_i24_e32 v4, s9, v4
395; GFX9-NEXT:    s_bfe_i32 s1, s2, 0x40010
396; GFX9-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
397; GFX9-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
398; GFX9-NEXT:    s_bfe_i32 s5, s2, 0x40014
399; GFX9-NEXT:    v_mov_b32_e32 v8, s1
400; GFX9-NEXT:    s_bfe_i32 s4, s0, 0x40010
401; GFX9-NEXT:    s_bfe_i32 s7, s2, 0x40018
402; GFX9-NEXT:    v_mov_b32_e32 v9, s5
403; GFX9-NEXT:    s_bfe_i32 s1, s0, 0x40014
404; GFX9-NEXT:    s_bfe_i32 s5, s0, 0x40018
405; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
406; GFX9-NEXT:    v_mov_b32_e32 v10, s7
407; GFX9-NEXT:    s_ashr_i32 s0, s0, 28
408; GFX9-NEXT:    s_waitcnt vmcnt(0)
409; GFX9-NEXT:    v_mad_i32_i24 v2, s6, v3, v2
410; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v5, v2
411; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
412; GFX9-NEXT:    v_mad_u32_u24 v2, v6, v7, v2
413; GFX9-NEXT:    v_mad_i32_i24 v2, s4, v8, v2
414; GFX9-NEXT:    v_mad_i32_i24 v2, s1, v9, v2
415; GFX9-NEXT:    v_mad_i32_i24 v2, s5, v10, v2
416; GFX9-NEXT:    v_mov_b32_e32 v3, s2
417; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
418; GFX9-NEXT:    global_store_short v[0:1], v2, off
419; GFX9-NEXT:    s_endpgm
420;
421; GFX9-DL-LABEL: idot8_acc16:
422; GFX9-DL:       ; %bb.0: ; %entry
423; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
424; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
425; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
426; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
427; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
428; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
429; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
430; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
431; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
432; GFX9-DL-NEXT:    s_bfe_i32 s1, s2, 0x40000
433; GFX9-DL-NEXT:    s_bfe_i32 s4, s2, 0x40004
434; GFX9-DL-NEXT:    s_bfe_i32 s5, s2, 0x40008
435; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
436; GFX9-DL-NEXT:    s_bfe_i32 s6, s0, 0x40000
437; GFX9-DL-NEXT:    s_lshr_b32 s1, s0, 12
438; GFX9-DL-NEXT:    s_lshr_b32 s7, s2, 12
439; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s5
440; GFX9-DL-NEXT:    s_bfe_i32 s9, s0, 0x40008
441; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s4
442; GFX9-DL-NEXT:    s_bfe_i32 s8, s0, 0x40004
443; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s1
444; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s7
445; GFX9-DL-NEXT:    v_mul_i32_i24_e32 v4, s9, v4
446; GFX9-DL-NEXT:    s_bfe_i32 s1, s2, 0x40010
447; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
448; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
449; GFX9-DL-NEXT:    s_bfe_i32 s5, s2, 0x40014
450; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s1
451; GFX9-DL-NEXT:    s_bfe_i32 s4, s0, 0x40010
452; GFX9-DL-NEXT:    s_bfe_i32 s7, s2, 0x40018
453; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s5
454; GFX9-DL-NEXT:    s_bfe_i32 s1, s0, 0x40014
455; GFX9-DL-NEXT:    s_bfe_i32 s5, s0, 0x40018
456; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 28
457; GFX9-DL-NEXT:    v_mov_b32_e32 v10, s7
458; GFX9-DL-NEXT:    s_ashr_i32 s0, s0, 28
459; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
460; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v3, v2
461; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v5, v2
462; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
463; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v6, v7, v2
464; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s4, v8, v2
465; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v9, v2
466; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s5, v10, v2
467; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
468; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
469; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
470; GFX9-DL-NEXT:    s_endpgm
471;
472; GFX10-DL-LABEL: idot8_acc16:
473; GFX10-DL:       ; %bb.0: ; %entry
474; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
475; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
476; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
477; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
478; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
479; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
480; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
481; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
482; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
483; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
484; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
485; GFX10-DL-NEXT:    s_lshr_b32 s4, s0, 12
486; GFX10-DL-NEXT:    s_lshr_b32 s5, s1, 12
487; GFX10-DL-NEXT:    s_bfe_i32 s6, s0, 0x40000
488; GFX10-DL-NEXT:    s_bfe_i32 s7, s1, 0x40000
489; GFX10-DL-NEXT:    s_bfe_i32 s8, s0, 0x40004
490; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s4
491; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s5
492; GFX10-DL-NEXT:    s_bfe_i32 s9, s1, 0x40004
493; GFX10-DL-NEXT:    s_bfe_i32 s10, s0, 0x40008
494; GFX10-DL-NEXT:    s_bfe_i32 s11, s1, 0x40008
495; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v3, 12, v3
496; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v4
497; GFX10-DL-NEXT:    s_bfe_i32 s4, s1, 0x40010
498; GFX10-DL-NEXT:    s_bfe_i32 s5, s0, 0x40014
499; GFX10-DL-NEXT:    v_mul_i32_i24_e64 v5, s10, s11
500; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
501; GFX10-DL-NEXT:    v_and_b32_e32 v4, s2, v4
502; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40010
503; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
504; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s6, s7, v2
505; GFX10-DL-NEXT:    s_bfe_i32 s6, s1, 0x40014
506; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s8, s9, v2
507; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
508; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
509; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s4, v2
510; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40018
511; GFX10-DL-NEXT:    s_bfe_i32 s4, s1, 0x40018
512; GFX10-DL-NEXT:    s_ashr_i32 s0, s0, 28
513; GFX10-DL-NEXT:    s_ashr_i32 s1, s1, 28
514; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s5, s6, v2
515; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s4, v2
516; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v2
517; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
518; GFX10-DL-NEXT:    s_endpgm
519                                       <8 x i4> addrspace(1)* %src2,
520                                       i16 addrspace(1)* nocapture %dst) {
521entry:
522  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
523  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
524
525  %v1e0 = extractelement <8 x i4> %vec1, i64 0
526  %cv1e0 = sext i4 %v1e0 to i16
527  %v2e0 = extractelement <8 x i4> %vec2, i64 0
528  %cv2e0 = sext i4 %v2e0 to i16
529  %mul0 = mul nuw nsw i16 %cv1e0, %cv2e0
530
531  %v1e1 = extractelement <8 x i4> %vec1, i64 1
532  %cv1e1 = sext i4 %v1e1 to i16
533  %v2e1 = extractelement <8 x i4> %vec2, i64 1
534  %cv2e1 = sext i4 %v2e1 to i16
535  %mul1 = mul nuw nsw i16 %cv1e1, %cv2e1
536
537  %v1e2 = extractelement <8 x i4> %vec1, i64 2
538  %cv1e2 = sext i4 %v1e2 to i16
539  %v2e2 = extractelement <8 x i4> %vec2, i64 2
540  %cv2e2 = sext i4 %v2e2 to i16
541  %mul2 = mul nuw nsw i16 %cv1e2, %cv2e2
542
543  %v1e3 = extractelement <8 x i4> %vec1, i64 3
544  %cv1e3 = sext i4 %v1e3 to i16
545  %v2e3 = extractelement <8 x i4> %vec2, i64 3
546  %cv2e3 = sext i4 %v2e3 to i16
547  %mul3 = mul nuw nsw i16 %cv1e3, %cv2e3
548
549  %v1e4 = extractelement <8 x i4> %vec1, i64 4
550  %cv1e4 = sext i4 %v1e4 to i16
551  %v2e4 = extractelement <8 x i4> %vec2, i64 4
552  %cv2e4 = sext i4 %v2e4 to i16
553  %mul4 = mul nuw nsw i16 %cv1e4, %cv2e4
554
555  %v1e5 = extractelement <8 x i4> %vec1, i64 5
556  %cv1e5 = sext i4 %v1e5 to i16
557  %v2e5 = extractelement <8 x i4> %vec2, i64 5
558  %cv2e5 = sext i4 %v2e5 to i16
559  %mul5 = mul nuw nsw i16 %cv1e5, %cv2e5
560
561  %v1e6 = extractelement <8 x i4> %vec1, i64 6
562  %cv1e6 = sext i4 %v1e6 to i16
563  %v2e6 = extractelement <8 x i4> %vec2, i64 6
564  %cv2e6 = sext i4 %v2e6 to i16
565  %mul6 = mul nuw nsw i16 %cv1e6, %cv2e6
566
567  %v1e7 = extractelement <8 x i4> %vec1, i64 7
568  %cv1e7 = sext i4 %v1e7 to i16
569  %v2e7 = extractelement <8 x i4> %vec2, i64 7
570  %cv2e7 = sext i4 %v2e7 to i16
571  %mul7 = mul nuw nsw i16 %cv1e7, %cv2e7
572
573  %acc = load i16, i16 addrspace(1)* %dst, align 4
574  %add1 = add i16 %mul0, %acc
575  %add2 = add i16 %add1, %mul1
576  %add3 = add i16 %add2, %mul2
577  %add4 = add i16 %add3, %mul3
578  %add5 = add i16 %add4, %mul4
579  %add6 = add i16 %add5, %mul5
580  %add7 = add i16 %add6, %mul6
581  %add8 = add i16 %add7, %mul7
582
583  store i16 %add8, i16 addrspace(1)* %dst, align 4
584  ret void
585}
586
587; TODO: Support this pattern.
588define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
589; GFX7-LABEL: idot8_acc8:
590; GFX7:       ; %bb.0: ; %entry
591; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
592; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
593; GFX7-NEXT:    s_mov_b32 s7, 0xf000
594; GFX7-NEXT:    s_mov_b32 s6, -1
595; GFX7-NEXT:    s_movk_i32 s0, 0xff
596; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
597; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
598; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
599; GFX7-NEXT:    s_load_dword s2, s[10:11], 0x0
600; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
601; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40000
602; GFX7-NEXT:    s_bfe_i32 s9, s2, 0x40000
603; GFX7-NEXT:    s_bfe_i32 s11, s2, 0x40004
604; GFX7-NEXT:    s_and_b32 s9, s9, s0
605; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40004
606; GFX7-NEXT:    s_bfe_i32 s13, s2, 0x40008
607; GFX7-NEXT:    s_and_b32 s11, s11, s0
608; GFX7-NEXT:    s_and_b32 s8, s8, s0
609; GFX7-NEXT:    v_mov_b32_e32 v1, s9
610; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40008
611; GFX7-NEXT:    s_bfe_i32 s15, s2, 0x4000c
612; GFX7-NEXT:    s_and_b32 s13, s13, s0
613; GFX7-NEXT:    s_and_b32 s10, s10, s0
614; GFX7-NEXT:    v_mov_b32_e32 v2, s11
615; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x4000c
616; GFX7-NEXT:    s_bfe_i32 s17, s2, 0x40010
617; GFX7-NEXT:    s_and_b32 s15, s15, s0
618; GFX7-NEXT:    s_and_b32 s12, s12, s0
619; GFX7-NEXT:    v_mov_b32_e32 v3, s13
620; GFX7-NEXT:    s_bfe_i32 s16, s1, 0x40010
621; GFX7-NEXT:    s_bfe_i32 s19, s2, 0x40014
622; GFX7-NEXT:    s_and_b32 s17, s17, s0
623; GFX7-NEXT:    s_and_b32 s14, s14, s0
624; GFX7-NEXT:    v_mov_b32_e32 v4, s15
625; GFX7-NEXT:    s_bfe_i32 s21, s2, 0x40018
626; GFX7-NEXT:    s_bfe_i32 s18, s1, 0x40014
627; GFX7-NEXT:    s_and_b32 s19, s19, s0
628; GFX7-NEXT:    s_and_b32 s16, s16, s0
629; GFX7-NEXT:    v_mov_b32_e32 v5, s17
630; GFX7-NEXT:    s_bfe_i32 s20, s1, 0x40018
631; GFX7-NEXT:    s_ashr_i32 s2, s2, 28
632; GFX7-NEXT:    s_and_b32 s21, s21, s0
633; GFX7-NEXT:    s_and_b32 s18, s18, s0
634; GFX7-NEXT:    v_mov_b32_e32 v6, s19
635; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
636; GFX7-NEXT:    s_and_b32 s20, s20, s0
637; GFX7-NEXT:    s_and_b32 s2, s2, s0
638; GFX7-NEXT:    v_mov_b32_e32 v7, s21
639; GFX7-NEXT:    s_and_b32 s0, s1, s0
640; GFX7-NEXT:    s_waitcnt vmcnt(0)
641; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
642; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v2, v0
643; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
644; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v4, v0
645; GFX7-NEXT:    v_mad_u32_u24 v0, s16, v5, v0
646; GFX7-NEXT:    v_mad_u32_u24 v0, s18, v6, v0
647; GFX7-NEXT:    v_mad_u32_u24 v0, s20, v7, v0
648; GFX7-NEXT:    v_mov_b32_e32 v1, s2
649; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
650; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
651; GFX7-NEXT:    s_endpgm
652;
653; GFX8-LABEL: idot8_acc8:
654; GFX8:       ; %bb.0: ; %entry
655; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
656; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
657; GFX8-NEXT:    s_movk_i32 s2, 0xff
658; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
659; GFX8-NEXT:    s_load_dword s6, s[6:7], 0x0
660; GFX8-NEXT:    v_mov_b32_e32 v0, s0
661; GFX8-NEXT:    v_mov_b32_e32 v1, s1
662; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
663; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
664; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
665; GFX8-NEXT:    s_bfe_i32 s7, s6, 0x40000
666; GFX8-NEXT:    s_lshr_b32 s4, s6, 12
667; GFX8-NEXT:    s_bfe_i32 s9, s6, 0x40004
668; GFX8-NEXT:    s_bfe_i32 s11, s6, 0x40008
669; GFX8-NEXT:    s_lshr_b32 s1, s0, 12
670; GFX8-NEXT:    s_bfe_i32 s5, s0, 0x40000
671; GFX8-NEXT:    v_mov_b32_e32 v6, s7
672; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
673; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s4
674; GFX8-NEXT:    s_bfe_i32 s8, s0, 0x40004
675; GFX8-NEXT:    s_bfe_i32 s10, s0, 0x40008
676; GFX8-NEXT:    v_mov_b32_e32 v3, s11
677; GFX8-NEXT:    v_mov_b32_e32 v7, s9
678; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
679; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
680; GFX8-NEXT:    v_mul_i32_i24_e32 v3, s10, v3
681; GFX8-NEXT:    s_bfe_i32 s13, s6, 0x40010
682; GFX8-NEXT:    v_and_b32_e32 v4, s2, v4
683; GFX8-NEXT:    v_and_b32_e32 v5, s2, v5
684; GFX8-NEXT:    s_bfe_i32 s15, s6, 0x40014
685; GFX8-NEXT:    s_bfe_i32 s12, s0, 0x40010
686; GFX8-NEXT:    v_mov_b32_e32 v8, s13
687; GFX8-NEXT:    s_bfe_i32 s17, s6, 0x40018
688; GFX8-NEXT:    s_bfe_i32 s14, s0, 0x40014
689; GFX8-NEXT:    v_mov_b32_e32 v9, s15
690; GFX8-NEXT:    s_bfe_i32 s16, s0, 0x40018
691; GFX8-NEXT:    s_ashr_i32 s6, s6, 28
692; GFX8-NEXT:    v_mov_b32_e32 v10, s17
693; GFX8-NEXT:    s_ashr_i32 s0, s0, 28
694; GFX8-NEXT:    s_waitcnt vmcnt(0)
695; GFX8-NEXT:    v_mad_i32_i24 v2, s5, v6, v2
696; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v7, v2
697; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
698; GFX8-NEXT:    v_mad_u32_u24 v2, v4, v5, v2
699; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v8, v2
700; GFX8-NEXT:    v_mad_i32_i24 v2, s14, v9, v2
701; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v10, v2
702; GFX8-NEXT:    v_mov_b32_e32 v3, s6
703; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
704; GFX8-NEXT:    flat_store_byte v[0:1], v2
705; GFX8-NEXT:    s_endpgm
706;
707; GFX9-LABEL: idot8_acc8:
708; GFX9:       ; %bb.0: ; %entry
709; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
710; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
711; GFX9-NEXT:    s_movk_i32 s2, 0xff
712; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
713; GFX9-NEXT:    s_load_dword s6, s[6:7], 0x0
714; GFX9-NEXT:    v_mov_b32_e32 v0, s0
715; GFX9-NEXT:    v_mov_b32_e32 v1, s1
716; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
717; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
718; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
719; GFX9-NEXT:    s_bfe_i32 s7, s6, 0x40000
720; GFX9-NEXT:    s_lshr_b32 s4, s6, 12
721; GFX9-NEXT:    s_bfe_i32 s9, s6, 0x40004
722; GFX9-NEXT:    s_bfe_i32 s11, s6, 0x40008
723; GFX9-NEXT:    s_lshr_b32 s1, s0, 12
724; GFX9-NEXT:    s_bfe_i32 s5, s0, 0x40000
725; GFX9-NEXT:    v_mov_b32_e32 v6, s7
726; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
727; GFX9-NEXT:    v_lshlrev_b16_e64 v5, 12, s4
728; GFX9-NEXT:    s_bfe_i32 s8, s0, 0x40004
729; GFX9-NEXT:    s_bfe_i32 s10, s0, 0x40008
730; GFX9-NEXT:    v_mov_b32_e32 v3, s11
731; GFX9-NEXT:    v_mov_b32_e32 v7, s9
732; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
733; GFX9-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
734; GFX9-NEXT:    v_mul_i32_i24_e32 v3, s10, v3
735; GFX9-NEXT:    s_bfe_i32 s13, s6, 0x40010
736; GFX9-NEXT:    v_and_b32_e32 v4, s2, v4
737; GFX9-NEXT:    v_and_b32_e32 v5, s2, v5
738; GFX9-NEXT:    s_bfe_i32 s15, s6, 0x40014
739; GFX9-NEXT:    s_bfe_i32 s12, s0, 0x40010
740; GFX9-NEXT:    v_mov_b32_e32 v8, s13
741; GFX9-NEXT:    s_bfe_i32 s17, s6, 0x40018
742; GFX9-NEXT:    s_bfe_i32 s14, s0, 0x40014
743; GFX9-NEXT:    v_mov_b32_e32 v9, s15
744; GFX9-NEXT:    s_bfe_i32 s16, s0, 0x40018
745; GFX9-NEXT:    s_ashr_i32 s6, s6, 28
746; GFX9-NEXT:    v_mov_b32_e32 v10, s17
747; GFX9-NEXT:    s_ashr_i32 s0, s0, 28
748; GFX9-NEXT:    s_waitcnt vmcnt(0)
749; GFX9-NEXT:    v_mad_i32_i24 v2, s5, v6, v2
750; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v7, v2
751; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
752; GFX9-NEXT:    v_mad_u32_u24 v2, v4, v5, v2
753; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v8, v2
754; GFX9-NEXT:    v_mad_i32_i24 v2, s14, v9, v2
755; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v10, v2
756; GFX9-NEXT:    v_mov_b32_e32 v3, s6
757; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
758; GFX9-NEXT:    global_store_byte v[0:1], v2, off
759; GFX9-NEXT:    s_endpgm
760;
761; GFX9-DL-LABEL: idot8_acc8:
762; GFX9-DL:       ; %bb.0: ; %entry
763; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
764; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
765; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
766; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
767; GFX9-DL-NEXT:    s_load_dword s6, s[6:7], 0x0
768; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
769; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
770; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
771; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
772; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
773; GFX9-DL-NEXT:    s_bfe_i32 s7, s6, 0x40000
774; GFX9-DL-NEXT:    s_lshr_b32 s4, s6, 12
775; GFX9-DL-NEXT:    s_bfe_i32 s9, s6, 0x40004
776; GFX9-DL-NEXT:    s_bfe_i32 s11, s6, 0x40008
777; GFX9-DL-NEXT:    s_lshr_b32 s1, s0, 12
778; GFX9-DL-NEXT:    s_bfe_i32 s5, s0, 0x40000
779; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s7
780; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
781; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s4
782; GFX9-DL-NEXT:    s_bfe_i32 s8, s0, 0x40004
783; GFX9-DL-NEXT:    s_bfe_i32 s10, s0, 0x40008
784; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s11
785; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s9
786; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
787; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
788; GFX9-DL-NEXT:    v_mul_i32_i24_e32 v3, s10, v3
789; GFX9-DL-NEXT:    s_bfe_i32 s13, s6, 0x40010
790; GFX9-DL-NEXT:    v_and_b32_e32 v4, s2, v4
791; GFX9-DL-NEXT:    v_and_b32_e32 v5, s2, v5
792; GFX9-DL-NEXT:    s_bfe_i32 s15, s6, 0x40014
793; GFX9-DL-NEXT:    s_bfe_i32 s12, s0, 0x40010
794; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s13
795; GFX9-DL-NEXT:    s_bfe_i32 s17, s6, 0x40018
796; GFX9-DL-NEXT:    s_bfe_i32 s14, s0, 0x40014
797; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s15
798; GFX9-DL-NEXT:    s_bfe_i32 s16, s0, 0x40018
799; GFX9-DL-NEXT:    s_ashr_i32 s6, s6, 28
800; GFX9-DL-NEXT:    v_mov_b32_e32 v10, s17
801; GFX9-DL-NEXT:    s_ashr_i32 s0, s0, 28
802; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
803; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s5, v6, v2
804; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v7, v2
805; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
806; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v4, v5, v2
807; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s12, v8, v2
808; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s14, v9, v2
809; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s16, v10, v2
810; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s6
811; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
812; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
813; GFX9-DL-NEXT:    s_endpgm
814;
815; GFX10-DL-LABEL: idot8_acc8:
816; GFX10-DL:       ; %bb.0: ; %entry
817; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
818; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
819; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
820; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
821; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
822; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
823; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
824; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
825; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
826; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
827; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
828; GFX10-DL-NEXT:    s_lshr_b32 s4, s0, 12
829; GFX10-DL-NEXT:    s_lshr_b32 s5, s1, 12
830; GFX10-DL-NEXT:    s_bfe_i32 s6, s0, 0x40000
831; GFX10-DL-NEXT:    s_bfe_i32 s7, s1, 0x40000
832; GFX10-DL-NEXT:    s_bfe_i32 s8, s0, 0x40004
833; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s4
834; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s5
835; GFX10-DL-NEXT:    s_bfe_i32 s9, s1, 0x40004
836; GFX10-DL-NEXT:    s_bfe_i32 s10, s0, 0x40008
837; GFX10-DL-NEXT:    s_bfe_i32 s11, s1, 0x40008
838; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v3, 12, v3
839; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v4
840; GFX10-DL-NEXT:    s_bfe_i32 s4, s1, 0x40010
841; GFX10-DL-NEXT:    s_bfe_i32 s5, s0, 0x40014
842; GFX10-DL-NEXT:    v_mul_i32_i24_e64 v5, s10, s11
843; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
844; GFX10-DL-NEXT:    v_and_b32_e32 v4, s2, v4
845; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40010
846; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
847; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s6, s7, v2
848; GFX10-DL-NEXT:    s_bfe_i32 s6, s1, 0x40014
849; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s8, s9, v2
850; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
851; GFX10-DL-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
852; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s4, v2
853; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40018
854; GFX10-DL-NEXT:    s_bfe_i32 s4, s1, 0x40018
855; GFX10-DL-NEXT:    s_ashr_i32 s0, s0, 28
856; GFX10-DL-NEXT:    s_ashr_i32 s1, s1, 28
857; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s5, s6, v2
858; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s4, v2
859; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v2
860; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
861; GFX10-DL-NEXT:    s_endpgm
862                                       <8 x i4> addrspace(1)* %src2,
863                                       i8 addrspace(1)* nocapture %dst) {
864entry:
865  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
866  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
867
868  %v1e0 = extractelement <8 x i4> %vec1, i64 0
869  %cv1e0 = sext i4 %v1e0 to i8
870  %v2e0 = extractelement <8 x i4> %vec2, i64 0
871  %cv2e0 = sext i4 %v2e0 to i8
872  %mul0 = mul nuw nsw i8 %cv1e0, %cv2e0
873
874  %v1e1 = extractelement <8 x i4> %vec1, i64 1
875  %cv1e1 = sext i4 %v1e1 to i8
876  %v2e1 = extractelement <8 x i4> %vec2, i64 1
877  %cv2e1 = sext i4 %v2e1 to i8
878  %mul1 = mul nuw nsw i8 %cv1e1, %cv2e1
879
880  %v1e2 = extractelement <8 x i4> %vec1, i64 2
881  %cv1e2 = sext i4 %v1e2 to i8
882  %v2e2 = extractelement <8 x i4> %vec2, i64 2
883  %cv2e2 = sext i4 %v2e2 to i8
884  %mul2 = mul nuw nsw i8 %cv1e2, %cv2e2
885
886  %v1e3 = extractelement <8 x i4> %vec1, i64 3
887  %cv1e3 = sext i4 %v1e3 to i8
888  %v2e3 = extractelement <8 x i4> %vec2, i64 3
889  %cv2e3 = sext i4 %v2e3 to i8
890  %mul3 = mul nuw nsw i8 %cv1e3, %cv2e3
891
892  %v1e4 = extractelement <8 x i4> %vec1, i64 4
893  %cv1e4 = sext i4 %v1e4 to i8
894  %v2e4 = extractelement <8 x i4> %vec2, i64 4
895  %cv2e4 = sext i4 %v2e4 to i8
896  %mul4 = mul nuw nsw i8 %cv1e4, %cv2e4
897
898  %v1e5 = extractelement <8 x i4> %vec1, i64 5
899  %cv1e5 = sext i4 %v1e5 to i8
900  %v2e5 = extractelement <8 x i4> %vec2, i64 5
901  %cv2e5 = sext i4 %v2e5 to i8
902  %mul5 = mul nuw nsw i8 %cv1e5, %cv2e5
903
904  %v1e6 = extractelement <8 x i4> %vec1, i64 6
905  %cv1e6 = sext i4 %v1e6 to i8
906  %v2e6 = extractelement <8 x i4> %vec2, i64 6
907  %cv2e6 = sext i4 %v2e6 to i8
908  %mul6 = mul nuw nsw i8 %cv1e6, %cv2e6
909
910  %v1e7 = extractelement <8 x i4> %vec1, i64 7
911  %cv1e7 = sext i4 %v1e7 to i8
912  %v2e7 = extractelement <8 x i4> %vec2, i64 7
913  %cv2e7 = sext i4 %v2e7 to i8
914  %mul7 = mul nuw nsw i8 %cv1e7, %cv2e7
915
916  %acc = load i8, i8 addrspace(1)* %dst, align 4
917  %add1 = add i8 %mul0, %acc
918  %add2 = add i8 %add1, %mul1
919  %add3 = add i8 %add2, %mul2
920  %add4 = add i8 %add3, %mul3
921  %add5 = add i8 %add4, %mul4
922  %add6 = add i8 %add5, %mul5
923  %add7 = add i8 %add6, %mul6
924  %add8 = add i8 %add7, %mul7
925
926  store i8 %add8, i8 addrspace(1)* %dst, align 4
927  ret void
928}
929
930; Make sure the pattern is not recognized if there are multiple uses of the
931; intermediate multiplications.
932define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
933; GFX7-LABEL: idot8_multiuses_mul1:
934; GFX7:       ; %bb.0: ; %entry
935; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
936; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
937; GFX7-NEXT:    s_mov_b32 s7, 0xf000
938; GFX7-NEXT:    s_mov_b32 s6, -1
939; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
940; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
941; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
942; GFX7-NEXT:    s_load_dword s21, s[4:5], 0x0
943; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
944; GFX7-NEXT:    s_bfe_i32 s2, s0, 0x40000
945; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40000
946; GFX7-NEXT:    v_mov_b32_e32 v0, s8
947; GFX7-NEXT:    v_mov_b32_e32 v1, s21
948; GFX7-NEXT:    v_mad_i32_i24 v1, s2, v0, v1
949; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40004
950; GFX7-NEXT:    s_bfe_i32 s9, s0, 0x40004
951; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40008
952; GFX7-NEXT:    v_mad_i32_i24 v0, s2, v0, v1
953; GFX7-NEXT:    v_mov_b32_e32 v2, s10
954; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v2, v0
955; GFX7-NEXT:    s_bfe_i32 s11, s0, 0x40008
956; GFX7-NEXT:    v_mov_b32_e32 v2, s12
957; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x4000c
958; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v2, v0
959; GFX7-NEXT:    s_bfe_i32 s13, s0, 0x4000c
960; GFX7-NEXT:    v_mov_b32_e32 v2, s14
961; GFX7-NEXT:    s_bfe_i32 s16, s1, 0x40010
962; GFX7-NEXT:    v_mad_i32_i24 v0, s13, v2, v0
963; GFX7-NEXT:    s_bfe_i32 s15, s0, 0x40010
964; GFX7-NEXT:    v_mov_b32_e32 v2, s16
965; GFX7-NEXT:    s_bfe_i32 s18, s1, 0x40014
966; GFX7-NEXT:    s_bfe_i32 s20, s1, 0x40018
967; GFX7-NEXT:    v_mad_i32_i24 v0, s15, v2, v0
968; GFX7-NEXT:    s_bfe_i32 s17, s0, 0x40014
969; GFX7-NEXT:    v_mov_b32_e32 v2, s18
970; GFX7-NEXT:    s_bfe_i32 s19, s0, 0x40018
971; GFX7-NEXT:    v_mad_i32_i24 v0, s17, v2, v0
972; GFX7-NEXT:    v_mov_b32_e32 v2, s20
973; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
974; GFX7-NEXT:    v_mad_i32_i24 v0, s19, v2, v0
975; GFX7-NEXT:    s_ashr_i32 s0, s0, 28
976; GFX7-NEXT:    v_mov_b32_e32 v2, s1
977; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v2, v0
978; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
979; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
980; GFX7-NEXT:    s_endpgm
981;
982; GFX8-LABEL: idot8_multiuses_mul1:
983; GFX8:       ; %bb.0: ; %entry
984; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
985; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
986; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
987; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
988; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
989; GFX8-NEXT:    s_load_dword s19, s[0:1], 0x0
990; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
991; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x40000
992; GFX8-NEXT:    s_bfe_i32 s6, s4, 0x40000
993; GFX8-NEXT:    v_mov_b32_e32 v0, s6
994; GFX8-NEXT:    v_mov_b32_e32 v1, s19
995; GFX8-NEXT:    v_mad_i32_i24 v1, s5, v0, v1
996; GFX8-NEXT:    s_bfe_i32 s8, s4, 0x40004
997; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x40004
998; GFX8-NEXT:    s_bfe_i32 s10, s4, 0x40008
999; GFX8-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
1000; GFX8-NEXT:    v_mov_b32_e32 v2, s8
1001; GFX8-NEXT:    v_mad_i32_i24 v0, s7, v2, v0
1002; GFX8-NEXT:    s_bfe_i32 s9, s2, 0x40008
1003; GFX8-NEXT:    v_mov_b32_e32 v2, s10
1004; GFX8-NEXT:    s_bfe_i32 s12, s4, 0x4000c
1005; GFX8-NEXT:    v_mad_i32_i24 v0, s9, v2, v0
1006; GFX8-NEXT:    s_bfe_i32 s11, s2, 0x4000c
1007; GFX8-NEXT:    v_mov_b32_e32 v2, s12
1008; GFX8-NEXT:    s_bfe_i32 s14, s4, 0x40010
1009; GFX8-NEXT:    v_mad_i32_i24 v0, s11, v2, v0
1010; GFX8-NEXT:    s_bfe_i32 s13, s2, 0x40010
1011; GFX8-NEXT:    v_mov_b32_e32 v2, s14
1012; GFX8-NEXT:    s_bfe_i32 s16, s4, 0x40014
1013; GFX8-NEXT:    s_bfe_i32 s18, s4, 0x40018
1014; GFX8-NEXT:    v_mad_i32_i24 v0, s13, v2, v0
1015; GFX8-NEXT:    s_bfe_i32 s15, s2, 0x40014
1016; GFX8-NEXT:    v_mov_b32_e32 v2, s16
1017; GFX8-NEXT:    s_bfe_i32 s17, s2, 0x40018
1018; GFX8-NEXT:    v_mad_i32_i24 v0, s15, v2, v0
1019; GFX8-NEXT:    v_mov_b32_e32 v2, s18
1020; GFX8-NEXT:    s_ashr_i32 s4, s4, 28
1021; GFX8-NEXT:    v_mad_i32_i24 v0, s17, v2, v0
1022; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
1023; GFX8-NEXT:    v_mov_b32_e32 v2, s4
1024; GFX8-NEXT:    v_mad_i32_i24 v0, s2, v2, v0
1025; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v1
1026; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1027; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1028; GFX8-NEXT:    flat_store_dword v[0:1], v2
1029; GFX8-NEXT:    s_endpgm
1030;
1031; GFX9-LABEL: idot8_multiuses_mul1:
1032; GFX9:       ; %bb.0: ; %entry
1033; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1034; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1035; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1036; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
1037; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
1038; GFX9-NEXT:    s_load_dword s19, s[0:1], 0x0
1039; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1040; GFX9-NEXT:    s_bfe_i32 s5, s2, 0x40000
1041; GFX9-NEXT:    s_bfe_i32 s6, s4, 0x40000
1042; GFX9-NEXT:    v_mov_b32_e32 v0, s6
1043; GFX9-NEXT:    v_mov_b32_e32 v1, s19
1044; GFX9-NEXT:    v_mad_i32_i24 v1, s5, v0, v1
1045; GFX9-NEXT:    s_bfe_i32 s8, s4, 0x40004
1046; GFX9-NEXT:    s_bfe_i32 s7, s2, 0x40004
1047; GFX9-NEXT:    s_bfe_i32 s10, s4, 0x40008
1048; GFX9-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
1049; GFX9-NEXT:    v_mov_b32_e32 v2, s8
1050; GFX9-NEXT:    v_mad_i32_i24 v0, s7, v2, v0
1051; GFX9-NEXT:    s_bfe_i32 s9, s2, 0x40008
1052; GFX9-NEXT:    v_mov_b32_e32 v2, s10
1053; GFX9-NEXT:    s_bfe_i32 s12, s4, 0x4000c
1054; GFX9-NEXT:    v_mad_i32_i24 v0, s9, v2, v0
1055; GFX9-NEXT:    s_bfe_i32 s11, s2, 0x4000c
1056; GFX9-NEXT:    v_mov_b32_e32 v2, s12
1057; GFX9-NEXT:    s_bfe_i32 s14, s4, 0x40010
1058; GFX9-NEXT:    v_mad_i32_i24 v0, s11, v2, v0
1059; GFX9-NEXT:    s_bfe_i32 s13, s2, 0x40010
1060; GFX9-NEXT:    v_mov_b32_e32 v2, s14
1061; GFX9-NEXT:    s_bfe_i32 s16, s4, 0x40014
1062; GFX9-NEXT:    s_bfe_i32 s18, s4, 0x40018
1063; GFX9-NEXT:    v_mad_i32_i24 v0, s13, v2, v0
1064; GFX9-NEXT:    s_bfe_i32 s15, s2, 0x40014
1065; GFX9-NEXT:    v_mov_b32_e32 v2, s16
1066; GFX9-NEXT:    s_bfe_i32 s17, s2, 0x40018
1067; GFX9-NEXT:    v_mad_i32_i24 v0, s15, v2, v0
1068; GFX9-NEXT:    v_mov_b32_e32 v2, s18
1069; GFX9-NEXT:    s_ashr_i32 s4, s4, 28
1070; GFX9-NEXT:    v_mad_i32_i24 v0, s17, v2, v0
1071; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
1072; GFX9-NEXT:    v_mov_b32_e32 v2, s4
1073; GFX9-NEXT:    v_mad_i32_i24 v0, s2, v2, v0
1074; GFX9-NEXT:    v_add_u32_e32 v2, v1, v0
1075; GFX9-NEXT:    v_mov_b32_e32 v0, s0
1076; GFX9-NEXT:    v_mov_b32_e32 v1, s1
1077; GFX9-NEXT:    global_store_dword v[0:1], v2, off
1078; GFX9-NEXT:    s_endpgm
1079;
1080; GFX9-DL-LABEL: idot8_multiuses_mul1:
1081; GFX9-DL:       ; %bb.0: ; %entry
1082; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1083; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1084; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1085; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1086; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1087; GFX9-DL-NEXT:    s_load_dword s19, s[0:1], 0x0
1088; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1089; GFX9-DL-NEXT:    s_bfe_i32 s5, s2, 0x40000
1090; GFX9-DL-NEXT:    s_bfe_i32 s6, s4, 0x40000
1091; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
1092; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s19
1093; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s5, v0, v1
1094; GFX9-DL-NEXT:    s_bfe_i32 s8, s4, 0x40004
1095; GFX9-DL-NEXT:    s_bfe_i32 s7, s2, 0x40004
1096; GFX9-DL-NEXT:    s_bfe_i32 s10, s4, 0x40008
1097; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
1098; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s8
1099; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s7, v2, v0
1100; GFX9-DL-NEXT:    s_bfe_i32 s9, s2, 0x40008
1101; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s10
1102; GFX9-DL-NEXT:    s_bfe_i32 s12, s4, 0x4000c
1103; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s9, v2, v0
1104; GFX9-DL-NEXT:    s_bfe_i32 s11, s2, 0x4000c
1105; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s12
1106; GFX9-DL-NEXT:    s_bfe_i32 s14, s4, 0x40010
1107; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s11, v2, v0
1108; GFX9-DL-NEXT:    s_bfe_i32 s13, s2, 0x40010
1109; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s14
1110; GFX9-DL-NEXT:    s_bfe_i32 s16, s4, 0x40014
1111; GFX9-DL-NEXT:    s_bfe_i32 s18, s4, 0x40018
1112; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s13, v2, v0
1113; GFX9-DL-NEXT:    s_bfe_i32 s15, s2, 0x40014
1114; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s16
1115; GFX9-DL-NEXT:    s_bfe_i32 s17, s2, 0x40018
1116; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s15, v2, v0
1117; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s18
1118; GFX9-DL-NEXT:    s_ashr_i32 s4, s4, 28
1119; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s17, v2, v0
1120; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 28
1121; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
1122; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s2, v2, v0
1123; GFX9-DL-NEXT:    v_add_u32_e32 v2, v1, v0
1124; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1125; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1126; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1127; GFX9-DL-NEXT:    s_endpgm
1128;
1129; GFX10-DL-LABEL: idot8_multiuses_mul1:
1130; GFX10-DL:       ; %bb.0: ; %entry
1131; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1132; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1133; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1134; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1135; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1136; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1137; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1138; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1139; GFX10-DL-NEXT:    s_bfe_i32 s6, s2, 0x40000
1140; GFX10-DL-NEXT:    s_bfe_i32 s7, s4, 0x40000
1141; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
1142; GFX10-DL-NEXT:    s_bfe_i32 s5, s2, 0x40004
1143; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x40004
1144; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s7, v0
1145; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s6, s7, v0
1146; GFX10-DL-NEXT:    s_bfe_i32 s6, s2, 0x40008
1147; GFX10-DL-NEXT:    s_bfe_i32 s7, s4, 0x40008
1148; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s5, s8, v1
1149; GFX10-DL-NEXT:    s_bfe_i32 s5, s2, 0x4000c
1150; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x4000c
1151; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s6, s7, v1
1152; GFX10-DL-NEXT:    s_bfe_i32 s6, s2, 0x40010
1153; GFX10-DL-NEXT:    s_bfe_i32 s7, s4, 0x40010
1154; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s5, s8, v1
1155; GFX10-DL-NEXT:    s_bfe_i32 s5, s2, 0x40014
1156; GFX10-DL-NEXT:    s_bfe_i32 s8, s4, 0x40014
1157; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s6, s7, v1
1158; GFX10-DL-NEXT:    s_bfe_i32 s6, s2, 0x40018
1159; GFX10-DL-NEXT:    s_bfe_i32 s7, s4, 0x40018
1160; GFX10-DL-NEXT:    s_ashr_i32 s2, s2, 28
1161; GFX10-DL-NEXT:    s_ashr_i32 s4, s4, 28
1162; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s5, s8, v1
1163; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s6, s7, v1
1164; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s4, v1
1165; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v0, v1
1166; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1167; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1168; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1169; GFX10-DL-NEXT:    s_endpgm
1170                                                <8 x i4> addrspace(1)* %src2,
1171                                                i32 addrspace(1)* nocapture %dst) {
1172entry:
1173  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1174  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1175
1176  %v1e0 = extractelement <8 x i4> %vec1, i64 0
1177  %cv1e0 = sext i4 %v1e0 to i32
1178  %v2e0 = extractelement <8 x i4> %vec2, i64 0
1179  %cv2e0 = sext i4 %v2e0 to i32
1180  %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
1181
1182  %v1e1 = extractelement <8 x i4> %vec1, i64 1
1183  %cv1e1 = sext i4 %v1e1 to i32
1184  %v2e1 = extractelement <8 x i4> %vec2, i64 1
1185  %cv2e1 = sext i4 %v2e1 to i32
1186  %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
1187
1188  %v1e2 = extractelement <8 x i4> %vec1, i64 2
1189  %cv1e2 = sext i4 %v1e2 to i32
1190  %v2e2 = extractelement <8 x i4> %vec2, i64 2
1191  %cv2e2 = sext i4 %v2e2 to i32
1192  %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
1193
1194  %v1e3 = extractelement <8 x i4> %vec1, i64 3
1195  %cv1e3 = sext i4 %v1e3 to i32
1196  %v2e3 = extractelement <8 x i4> %vec2, i64 3
1197  %cv2e3 = sext i4 %v2e3 to i32
1198  %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
1199
1200  %v1e4 = extractelement <8 x i4> %vec1, i64 4
1201  %cv1e4 = sext i4 %v1e4 to i32
1202  %v2e4 = extractelement <8 x i4> %vec2, i64 4
1203  %cv2e4 = sext i4 %v2e4 to i32
1204  %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
1205
1206  %v1e5 = extractelement <8 x i4> %vec1, i64 5
1207  %cv1e5 = sext i4 %v1e5 to i32
1208  %v2e5 = extractelement <8 x i4> %vec2, i64 5
1209  %cv2e5 = sext i4 %v2e5 to i32
1210  %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
1211
1212  %v1e6 = extractelement <8 x i4> %vec1, i64 6
1213  %cv1e6 = sext i4 %v1e6 to i32
1214  %v2e6 = extractelement <8 x i4> %vec2, i64 6
1215  %cv2e6 = sext i4 %v2e6 to i32
1216  %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
1217
1218  %v1e7 = extractelement <8 x i4> %vec1, i64 7
1219  %cv1e7 = sext i4 %v1e7 to i32
1220  %v2e7 = extractelement <8 x i4> %vec2, i64 7
1221  %cv2e7 = sext i4 %v2e7 to i32
1222  %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
1223
1224  %acc = load i32, i32 addrspace(1)* %dst, align 4
1225  %add =  add i32  %mul0, %acc
1226  %add1 = add i32 %mul0, %add
1227  %add2 = add i32 %add1, %mul1
1228  %add3 = add i32 %add2, %mul2
1229  %add4 = add i32 %add3, %mul3
1230  %add5 = add i32 %add4, %mul4
1231  %add6 = add i32 %add5, %mul5
1232  %add7 = add i32 %add6, %mul6
1233  %add8 = add i32 %add7, %mul7
1234
1235  %res = add i32 %add, %add8
1236  store i32 %res, i32 addrspace(1)* %dst, align 4
1237  ret void
1238}
1239
1240; TODO: Support this pattern.
1241define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
1242; GFX7-LABEL: idot8_acc32_vecMul:
1243; GFX7:       ; %bb.0: ; %entry
1244; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
1245; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
1246; GFX7-NEXT:    s_mov_b32 s7, 0xf000
1247; GFX7-NEXT:    s_mov_b32 s6, -1
1248; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1249; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
1250; GFX7-NEXT:    s_load_dword s9, s[10:11], 0x0
1251; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1252; GFX7-NEXT:    s_ashr_i64 s[10:11], s[0:1], 60
1253; GFX7-NEXT:    s_lshl_b32 s11, s1, 4
1254; GFX7-NEXT:    s_ashr_i64 s[16:17], s[10:11], 60
1255; GFX7-NEXT:    s_lshl_b32 s11, s1, 16
1256; GFX7-NEXT:    s_ashr_i64 s[18:19], s[10:11], 60
1257; GFX7-NEXT:    s_lshl_b32 s11, s1, 20
1258; GFX7-NEXT:    s_lshl_b32 s13, s1, 8
1259; GFX7-NEXT:    s_lshl_b32 s15, s1, 12
1260; GFX7-NEXT:    s_ashr_i64 s[20:21], s[10:11], 60
1261; GFX7-NEXT:    s_lshl_b32 s11, s1, 24
1262; GFX7-NEXT:    s_lshl_b32 s1, s1, 28
1263; GFX7-NEXT:    s_ashr_i64 s[0:1], s[0:1], 60
1264; GFX7-NEXT:    s_lshl_b32 s1, s9, 4
1265; GFX7-NEXT:    s_ashr_i64 s[26:27], s[0:1], 60
1266; GFX7-NEXT:    s_lshl_b32 s1, s9, 8
1267; GFX7-NEXT:    s_ashr_i64 s[28:29], s[0:1], 60
1268; GFX7-NEXT:    s_lshl_b32 s1, s9, 12
1269; GFX7-NEXT:    s_ashr_i64 s[30:31], s[0:1], 60
1270; GFX7-NEXT:    s_lshl_b32 s1, s9, 16
1271; GFX7-NEXT:    s_ashr_i64 s[32:33], s[0:1], 60
1272; GFX7-NEXT:    s_lshl_b32 s1, s9, 20
1273; GFX7-NEXT:    s_ashr_i64 s[34:35], s[0:1], 60
1274; GFX7-NEXT:    s_lshl_b32 s1, s9, 24
1275; GFX7-NEXT:    s_ashr_i64 s[36:37], s[0:1], 60
1276; GFX7-NEXT:    s_lshl_b32 s1, s9, 28
1277; GFX7-NEXT:    s_ashr_i64 s[24:25], s[8:9], 60
1278; GFX7-NEXT:    s_ashr_i64 s[8:9], s[0:1], 60
1279; GFX7-NEXT:    s_load_dword s1, s[4:5], 0x0
1280; GFX7-NEXT:    v_mov_b32_e32 v0, s8
1281; GFX7-NEXT:    s_ashr_i64 s[22:23], s[10:11], 60
1282; GFX7-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
1283; GFX7-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
1284; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1285; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1286; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v0, v1
1287; GFX7-NEXT:    v_mov_b32_e32 v1, s36
1288; GFX7-NEXT:    v_mad_i32_i24 v0, s22, v1, v0
1289; GFX7-NEXT:    v_mov_b32_e32 v1, s34
1290; GFX7-NEXT:    v_mad_i32_i24 v0, s20, v1, v0
1291; GFX7-NEXT:    v_mov_b32_e32 v1, s32
1292; GFX7-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
1293; GFX7-NEXT:    v_mov_b32_e32 v1, s30
1294; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
1295; GFX7-NEXT:    v_mov_b32_e32 v1, s28
1296; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
1297; GFX7-NEXT:    v_mov_b32_e32 v1, s26
1298; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
1299; GFX7-NEXT:    v_mov_b32_e32 v1, s24
1300; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
1301; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1302; GFX7-NEXT:    s_endpgm
1303;
1304; GFX8-LABEL: idot8_acc32_vecMul:
1305; GFX8:       ; %bb.0: ; %entry
1306; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1307; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1308; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1309; GFX8-NEXT:    s_load_dword s5, s[4:5], 0x0
1310; GFX8-NEXT:    s_load_dword s7, s[6:7], 0x0
1311; GFX8-NEXT:    s_load_dword s2, s[0:1], 0x0
1312; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1313; GFX8-NEXT:    s_ashr_i64 s[8:9], s[4:5], 60
1314; GFX8-NEXT:    s_lshl_b32 s9, s5, 4
1315; GFX8-NEXT:    s_ashr_i64 s[16:17], s[8:9], 60
1316; GFX8-NEXT:    s_lshl_b32 s9, s5, 20
1317; GFX8-NEXT:    s_lshl_b32 s11, s5, 8
1318; GFX8-NEXT:    s_lshl_b32 s13, s5, 12
1319; GFX8-NEXT:    s_lshl_b32 s15, s5, 16
1320; GFX8-NEXT:    s_ashr_i64 s[18:19], s[8:9], 60
1321; GFX8-NEXT:    s_lshl_b32 s9, s5, 24
1322; GFX8-NEXT:    s_lshl_b32 s5, s5, 28
1323; GFX8-NEXT:    s_ashr_i64 s[4:5], s[4:5], 60
1324; GFX8-NEXT:    s_lshl_b32 s5, s7, 4
1325; GFX8-NEXT:    s_ashr_i64 s[24:25], s[4:5], 60
1326; GFX8-NEXT:    s_lshl_b32 s5, s7, 8
1327; GFX8-NEXT:    s_ashr_i64 s[26:27], s[4:5], 60
1328; GFX8-NEXT:    s_lshl_b32 s5, s7, 12
1329; GFX8-NEXT:    s_ashr_i64 s[28:29], s[4:5], 60
1330; GFX8-NEXT:    s_lshl_b32 s5, s7, 16
1331; GFX8-NEXT:    s_ashr_i64 s[30:31], s[4:5], 60
1332; GFX8-NEXT:    s_lshl_b32 s5, s7, 20
1333; GFX8-NEXT:    s_ashr_i64 s[32:33], s[4:5], 60
1334; GFX8-NEXT:    s_lshl_b32 s5, s7, 24
1335; GFX8-NEXT:    s_ashr_i64 s[34:35], s[4:5], 60
1336; GFX8-NEXT:    s_lshl_b32 s5, s7, 28
1337; GFX8-NEXT:    s_ashr_i64 s[22:23], s[6:7], 60
1338; GFX8-NEXT:    s_ashr_i64 s[6:7], s[4:5], 60
1339; GFX8-NEXT:    v_mov_b32_e32 v0, s6
1340; GFX8-NEXT:    v_mov_b32_e32 v1, s2
1341; GFX8-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
1342; GFX8-NEXT:    s_ashr_i64 s[20:21], s[8:9], 60
1343; GFX8-NEXT:    v_mov_b32_e32 v1, s34
1344; GFX8-NEXT:    v_mad_i32_i24 v0, s20, v1, v0
1345; GFX8-NEXT:    v_mov_b32_e32 v1, s32
1346; GFX8-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
1347; GFX8-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
1348; GFX8-NEXT:    v_mov_b32_e32 v1, s30
1349; GFX8-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
1350; GFX8-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
1351; GFX8-NEXT:    v_mov_b32_e32 v1, s28
1352; GFX8-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
1353; GFX8-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
1354; GFX8-NEXT:    v_mov_b32_e32 v1, s26
1355; GFX8-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
1356; GFX8-NEXT:    v_mov_b32_e32 v1, s24
1357; GFX8-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
1358; GFX8-NEXT:    v_mov_b32_e32 v1, s22
1359; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v1, v0
1360; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1361; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1362; GFX8-NEXT:    flat_store_dword v[0:1], v2
1363; GFX8-NEXT:    s_endpgm
1364;
1365; GFX9-LABEL: idot8_acc32_vecMul:
1366; GFX9:       ; %bb.0: ; %entry
1367; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1368; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1369; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1370; GFX9-NEXT:    s_load_dword s5, s[4:5], 0x0
1371; GFX9-NEXT:    s_load_dword s7, s[6:7], 0x0
1372; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x0
1373; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1374; GFX9-NEXT:    s_ashr_i64 s[8:9], s[4:5], 60
1375; GFX9-NEXT:    s_lshl_b32 s9, s5, 4
1376; GFX9-NEXT:    s_ashr_i64 s[16:17], s[8:9], 60
1377; GFX9-NEXT:    s_lshl_b32 s9, s5, 20
1378; GFX9-NEXT:    s_lshl_b32 s11, s5, 8
1379; GFX9-NEXT:    s_lshl_b32 s13, s5, 12
1380; GFX9-NEXT:    s_lshl_b32 s15, s5, 16
1381; GFX9-NEXT:    s_ashr_i64 s[18:19], s[8:9], 60
1382; GFX9-NEXT:    s_lshl_b32 s9, s5, 24
1383; GFX9-NEXT:    s_lshl_b32 s5, s5, 28
1384; GFX9-NEXT:    s_ashr_i64 s[4:5], s[4:5], 60
1385; GFX9-NEXT:    s_lshl_b32 s5, s7, 4
1386; GFX9-NEXT:    s_ashr_i64 s[24:25], s[4:5], 60
1387; GFX9-NEXT:    s_lshl_b32 s5, s7, 8
1388; GFX9-NEXT:    s_ashr_i64 s[26:27], s[4:5], 60
1389; GFX9-NEXT:    s_lshl_b32 s5, s7, 12
1390; GFX9-NEXT:    s_ashr_i64 s[28:29], s[4:5], 60
1391; GFX9-NEXT:    s_lshl_b32 s5, s7, 16
1392; GFX9-NEXT:    s_ashr_i64 s[30:31], s[4:5], 60
1393; GFX9-NEXT:    s_lshl_b32 s5, s7, 20
1394; GFX9-NEXT:    s_ashr_i64 s[32:33], s[4:5], 60
1395; GFX9-NEXT:    s_lshl_b32 s5, s7, 24
1396; GFX9-NEXT:    s_ashr_i64 s[34:35], s[4:5], 60
1397; GFX9-NEXT:    s_lshl_b32 s5, s7, 28
1398; GFX9-NEXT:    s_ashr_i64 s[22:23], s[6:7], 60
1399; GFX9-NEXT:    s_ashr_i64 s[6:7], s[4:5], 60
1400; GFX9-NEXT:    v_mov_b32_e32 v0, s6
1401; GFX9-NEXT:    v_mov_b32_e32 v1, s2
1402; GFX9-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
1403; GFX9-NEXT:    s_ashr_i64 s[20:21], s[8:9], 60
1404; GFX9-NEXT:    v_mov_b32_e32 v1, s34
1405; GFX9-NEXT:    v_mad_i32_i24 v0, s20, v1, v0
1406; GFX9-NEXT:    v_mov_b32_e32 v1, s32
1407; GFX9-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
1408; GFX9-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
1409; GFX9-NEXT:    v_mov_b32_e32 v1, s30
1410; GFX9-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
1411; GFX9-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
1412; GFX9-NEXT:    v_mov_b32_e32 v1, s28
1413; GFX9-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
1414; GFX9-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
1415; GFX9-NEXT:    v_mov_b32_e32 v1, s26
1416; GFX9-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
1417; GFX9-NEXT:    v_mov_b32_e32 v1, s24
1418; GFX9-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
1419; GFX9-NEXT:    v_mov_b32_e32 v1, s22
1420; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v1, v0
1421; GFX9-NEXT:    v_mov_b32_e32 v0, s0
1422; GFX9-NEXT:    v_mov_b32_e32 v1, s1
1423; GFX9-NEXT:    global_store_dword v[0:1], v2, off
1424; GFX9-NEXT:    s_endpgm
1425;
1426; GFX9-DL-LABEL: idot8_acc32_vecMul:
1427; GFX9-DL:       ; %bb.0: ; %entry
1428; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1429; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1430; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1431; GFX9-DL-NEXT:    s_load_dword s5, s[4:5], 0x0
1432; GFX9-DL-NEXT:    s_load_dword s7, s[6:7], 0x0
1433; GFX9-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
1434; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1435; GFX9-DL-NEXT:    s_ashr_i64 s[8:9], s[4:5], 60
1436; GFX9-DL-NEXT:    s_lshl_b32 s9, s5, 4
1437; GFX9-DL-NEXT:    s_ashr_i64 s[16:17], s[8:9], 60
1438; GFX9-DL-NEXT:    s_lshl_b32 s9, s5, 20
1439; GFX9-DL-NEXT:    s_lshl_b32 s11, s5, 8
1440; GFX9-DL-NEXT:    s_lshl_b32 s13, s5, 12
1441; GFX9-DL-NEXT:    s_lshl_b32 s15, s5, 16
1442; GFX9-DL-NEXT:    s_ashr_i64 s[18:19], s[8:9], 60
1443; GFX9-DL-NEXT:    s_lshl_b32 s9, s5, 24
1444; GFX9-DL-NEXT:    s_lshl_b32 s5, s5, 28
1445; GFX9-DL-NEXT:    s_ashr_i64 s[4:5], s[4:5], 60
1446; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 4
1447; GFX9-DL-NEXT:    s_ashr_i64 s[24:25], s[4:5], 60
1448; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 8
1449; GFX9-DL-NEXT:    s_ashr_i64 s[26:27], s[4:5], 60
1450; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 12
1451; GFX9-DL-NEXT:    s_ashr_i64 s[28:29], s[4:5], 60
1452; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 16
1453; GFX9-DL-NEXT:    s_ashr_i64 s[30:31], s[4:5], 60
1454; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 20
1455; GFX9-DL-NEXT:    s_ashr_i64 s[32:33], s[4:5], 60
1456; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 24
1457; GFX9-DL-NEXT:    s_ashr_i64 s[34:35], s[4:5], 60
1458; GFX9-DL-NEXT:    s_lshl_b32 s5, s7, 28
1459; GFX9-DL-NEXT:    s_ashr_i64 s[22:23], s[6:7], 60
1460; GFX9-DL-NEXT:    s_ashr_i64 s[6:7], s[4:5], 60
1461; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
1462; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
1463; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
1464; GFX9-DL-NEXT:    s_ashr_i64 s[20:21], s[8:9], 60
1465; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s34
1466; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s20, v1, v0
1467; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s32
1468; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
1469; GFX9-DL-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
1470; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s30
1471; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
1472; GFX9-DL-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
1473; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s28
1474; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
1475; GFX9-DL-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
1476; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s26
1477; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
1478; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s24
1479; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
1480; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s22
1481; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v1, v0
1482; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1483; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1484; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1485; GFX9-DL-NEXT:    s_endpgm
1486;
1487; GFX10-DL-LABEL: idot8_acc32_vecMul:
1488; GFX10-DL:       ; %bb.0: ; %entry
1489; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1490; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1491; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1492; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1493; GFX10-DL-NEXT:    s_load_dword s5, s[4:5], 0x0
1494; GFX10-DL-NEXT:    s_load_dword s7, s[6:7], 0x0
1495; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
1496; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1497; GFX10-DL-NEXT:    s_lshl_b32 s9, s5, 28
1498; GFX10-DL-NEXT:    s_lshl_b32 s11, s7, 28
1499; GFX10-DL-NEXT:    s_lshl_b32 s13, s5, 24
1500; GFX10-DL-NEXT:    s_lshl_b32 s15, s7, 24
1501; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
1502; GFX10-DL-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
1503; GFX10-DL-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
1504; GFX10-DL-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
1505; GFX10-DL-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
1506; GFX10-DL-NEXT:    s_lshl_b32 s9, s5, 20
1507; GFX10-DL-NEXT:    s_lshl_b32 s11, s7, 20
1508; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s8, s10, v0
1509; GFX10-DL-NEXT:    s_lshl_b32 s13, s5, 16
1510; GFX10-DL-NEXT:    s_lshl_b32 s15, s7, 16
1511; GFX10-DL-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
1512; GFX10-DL-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
1513; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s12, s14, v0
1514; GFX10-DL-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
1515; GFX10-DL-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
1516; GFX10-DL-NEXT:    s_lshl_b32 s9, s5, 12
1517; GFX10-DL-NEXT:    s_lshl_b32 s11, s7, 12
1518; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s8, s10, v0
1519; GFX10-DL-NEXT:    s_lshl_b32 s13, s5, 8
1520; GFX10-DL-NEXT:    s_lshl_b32 s15, s7, 8
1521; GFX10-DL-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
1522; GFX10-DL-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
1523; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s12, s14, v0
1524; GFX10-DL-NEXT:    s_lshl_b32 s9, s5, 4
1525; GFX10-DL-NEXT:    s_lshl_b32 s11, s7, 4
1526; GFX10-DL-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
1527; GFX10-DL-NEXT:    s_ashr_i64 s[14:15], s[14:15], 60
1528; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s8, s10, v0
1529; GFX10-DL-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
1530; GFX10-DL-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
1531; GFX10-DL-NEXT:    s_ashr_i64 s[4:5], s[4:5], 60
1532; GFX10-DL-NEXT:    s_ashr_i64 s[6:7], s[6:7], 60
1533; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s12, s14, v0
1534; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s8, s10, v0
1535; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s4, s6, v0
1536; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1537; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1538; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1539; GFX10-DL-NEXT:    s_endpgm
1540                                              <8 x i4> addrspace(1)* %src2,
1541                                              i32 addrspace(1)* nocapture %dst) {
1542entry:
1543  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1544  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1545
1546  %cvec1 = sext <8 x i4> %vec1 to <8 x i32>
1547  %cvec2 = sext <8 x i4> %vec2 to <8 x i32>
1548
1549  %mul = mul <8 x i32> %cvec1, %cvec2
1550  %mul0 = extractelement <8 x i32> %mul, i64 0
1551  %mul1 = extractelement <8 x i32> %mul, i64 1
1552  %mul2 = extractelement <8 x i32> %mul, i64 2
1553  %mul3 = extractelement <8 x i32> %mul, i64 3
1554  %mul4 = extractelement <8 x i32> %mul, i64 4
1555  %mul5 = extractelement <8 x i32> %mul, i64 5
1556  %mul6 = extractelement <8 x i32> %mul, i64 6
1557  %mul7 = extractelement <8 x i32> %mul, i64 7
1558
1559  %acc = load i32, i32 addrspace(1)* %dst, align 4
1560  %add1 = add i32 %mul0, %acc
1561  %add2 = add i32 %add1, %mul1
1562  %add3 = add i32 %add2, %mul2
1563  %add4 = add i32 %add3, %mul3
1564  %add5 = add i32 %add4, %mul4
1565  %add6 = add i32 %add5, %mul5
1566  %add7 = add i32 %add6, %mul6
1567  %add8 = add i32 %add7, %mul7
1568
1569  store i32 %add8, i32 addrspace(1)* %dst, align 4
1570  ret void
1571}
1572
1573; TODO: Support this pattern.
1574define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
1575; GFX7-LABEL: idot8_acc16_vecMul:
1576; GFX7:       ; %bb.0: ; %entry
1577; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
1578; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
1579; GFX7-NEXT:    s_mov_b32 s7, 0xf000
1580; GFX7-NEXT:    s_mov_b32 s6, -1
1581; GFX7-NEXT:    s_mov_b32 s2, 0xffff
1582; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1583; GFX7-NEXT:    s_load_dword s0, s[10:11], 0x0
1584; GFX7-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
1585; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
1586; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1587; GFX7-NEXT:    s_bfe_i32 s16, s0, 0x40018
1588; GFX7-NEXT:    s_bfe_i32 s17, s0, 0x40014
1589; GFX7-NEXT:    s_bfe_i32 s18, s0, 0x40010
1590; GFX7-NEXT:    s_bfe_i32 s19, s0, 0x40000
1591; GFX7-NEXT:    s_bfe_i32 s20, s0, 0x40004
1592; GFX7-NEXT:    s_bfe_i32 s21, s0, 0x40008
1593; GFX7-NEXT:    s_ashr_i32 s15, s0, 28
1594; GFX7-NEXT:    s_bfe_i32 s0, s0, 0x4000c
1595; GFX7-NEXT:    s_ashr_i32 s8, s1, 28
1596; GFX7-NEXT:    s_bfe_i32 s9, s1, 0x40018
1597; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40014
1598; GFX7-NEXT:    s_bfe_i32 s11, s1, 0x40010
1599; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40000
1600; GFX7-NEXT:    v_mov_b32_e32 v4, s19
1601; GFX7-NEXT:    s_bfe_i32 s13, s1, 0x40004
1602; GFX7-NEXT:    v_mov_b32_e32 v3, s20
1603; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x40008
1604; GFX7-NEXT:    v_mov_b32_e32 v2, s21
1605; GFX7-NEXT:    s_bfe_i32 s1, s1, 0x4000c
1606; GFX7-NEXT:    v_mov_b32_e32 v1, s0
1607; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s1, v1
1608; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s14, v2
1609; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s13, v3
1610; GFX7-NEXT:    v_mul_i32_i24_e32 v4, s12, v4
1611; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
1612; GFX7-NEXT:    v_and_b32_e32 v2, s2, v2
1613; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
1614; GFX7-NEXT:    v_and_b32_e32 v4, s2, v4
1615; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
1616; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
1617; GFX7-NEXT:    v_alignbit_b32 v3, v1, v2, 16
1618; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
1619; GFX7-NEXT:    v_mov_b32_e32 v5, s18
1620; GFX7-NEXT:    v_mov_b32_e32 v6, s17
1621; GFX7-NEXT:    v_mov_b32_e32 v7, s16
1622; GFX7-NEXT:    s_waitcnt vmcnt(0)
1623; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
1624; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
1625; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
1626; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
1627; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v5, v0
1628; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v6, v0
1629; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v7, v0
1630; GFX7-NEXT:    v_mov_b32_e32 v1, s15
1631; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
1632; GFX7-NEXT:    buffer_store_short v0, off, s[4:7], 0
1633; GFX7-NEXT:    s_endpgm
1634;
1635; GFX8-LABEL: idot8_acc16_vecMul:
1636; GFX8:       ; %bb.0: ; %entry
1637; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1638; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1639; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1640; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
1641; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1642; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1643; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
1644; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
1645; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1646; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s2
1647; GFX8-NEXT:    s_lshr_b32 s15, s2, 4
1648; GFX8-NEXT:    s_lshr_b32 s16, s2, 8
1649; GFX8-NEXT:    v_lshlrev_b16_e64 v12, 12, s16
1650; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
1651; GFX8-NEXT:    s_lshr_b32 s8, s0, 4
1652; GFX8-NEXT:    s_lshr_b32 s9, s0, 8
1653; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s9
1654; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s8
1655; GFX8-NEXT:    v_lshlrev_b16_e64 v13, 12, s15
1656; GFX8-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
1657; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
1658; GFX8-NEXT:    s_lshr_b32 s7, s0, 12
1659; GFX8-NEXT:    s_lshr_b32 s14, s2, 12
1660; GFX8-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
1661; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
1662; GFX8-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
1663; GFX8-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
1664; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s7
1665; GFX8-NEXT:    v_lshlrev_b16_e64 v14, 12, s14
1666; GFX8-NEXT:    s_lshr_b32 s6, s0, 16
1667; GFX8-NEXT:    s_lshr_b32 s13, s2, 16
1668; GFX8-NEXT:    v_mul_u32_u24_e32 v5, v5, v12
1669; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s6
1670; GFX8-NEXT:    v_lshlrev_b16_e64 v15, 12, s13
1671; GFX8-NEXT:    s_lshr_b32 s5, s0, 20
1672; GFX8-NEXT:    s_lshr_b32 s12, s2, 20
1673; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
1674; GFX8-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
1675; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s5
1676; GFX8-NEXT:    v_lshlrev_b16_e64 v16, 12, s12
1677; GFX8-NEXT:    s_lshr_b32 s4, s0, 24
1678; GFX8-NEXT:    s_lshr_b32 s11, s2, 24
1679; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
1680; GFX8-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
1681; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
1682; GFX8-NEXT:    v_lshlrev_b16_e64 v17, 12, s11
1683; GFX8-NEXT:    s_lshr_b32 s1, s0, 28
1684; GFX8-NEXT:    s_lshr_b32 s10, s2, 28
1685; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
1686; GFX8-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
1687; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s1
1688; GFX8-NEXT:    v_lshlrev_b16_e64 v18, 12, s10
1689; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
1690; GFX8-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
1691; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
1692; GFX8-NEXT:    v_ashrrev_i16_e32 v18, 12, v18
1693; GFX8-NEXT:    s_waitcnt vmcnt(0)
1694; GFX8-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
1695; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v13, v2
1696; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
1697; GFX8-NEXT:    v_mad_u32_u24 v2, v7, v14, v2
1698; GFX8-NEXT:    v_mad_u32_u24 v2, v8, v15, v2
1699; GFX8-NEXT:    v_mad_u32_u24 v2, v9, v16, v2
1700; GFX8-NEXT:    v_mad_u32_u24 v2, v10, v17, v2
1701; GFX8-NEXT:    v_mad_u32_u24 v2, v11, v18, v2
1702; GFX8-NEXT:    flat_store_short v[0:1], v2
1703; GFX8-NEXT:    s_endpgm
1704;
1705; GFX9-LABEL: idot8_acc16_vecMul:
1706; GFX9:       ; %bb.0: ; %entry
1707; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1708; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1709; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1710; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
1711; GFX9-NEXT:    s_load_dword s6, s[6:7], 0x0
1712; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1713; GFX9-NEXT:    s_and_b32 s4, s2, 15
1714; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40004
1715; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40008
1716; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x4000c
1717; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x40010
1718; GFX9-NEXT:    s_bfe_u32 s11, s2, 0x40018
1719; GFX9-NEXT:    s_lshr_b32 s12, s2, 28
1720; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x40014
1721; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s10, s2
1722; GFX9-NEXT:    v_pk_lshlrev_b16 v2, 12, s2 op_sel_hi:[0,1]
1723; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s11, s12
1724; GFX9-NEXT:    s_bfe_u32 s7, s6, 0x40018
1725; GFX9-NEXT:    s_lshr_b32 s13, s6, 28
1726; GFX9-NEXT:    s_bfe_u32 s14, s6, 0x40010
1727; GFX9-NEXT:    s_bfe_u32 s15, s6, 0x40014
1728; GFX9-NEXT:    s_bfe_u32 s16, s6, 0x40008
1729; GFX9-NEXT:    s_bfe_u32 s17, s6, 0x4000c
1730; GFX9-NEXT:    s_and_b32 s18, s6, 15
1731; GFX9-NEXT:    s_bfe_u32 s6, s6, 0x40004
1732; GFX9-NEXT:    v_pk_lshlrev_b16 v3, 12, s2 op_sel_hi:[0,1]
1733; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
1734; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s18, s6
1735; GFX9-NEXT:    v_pk_lshlrev_b16 v0, 12, s4 op_sel_hi:[0,1]
1736; GFX9-NEXT:    v_pk_lshlrev_b16 v4, 12, s2 op_sel_hi:[0,1]
1737; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s8, s9
1738; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s16, s17
1739; GFX9-NEXT:    v_pk_lshlrev_b16 v5, 12, s2 op_sel_hi:[0,1]
1740; GFX9-NEXT:    v_pk_lshlrev_b16 v1, 12, s4 op_sel_hi:[0,1]
1741; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s14, s15
1742; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 12, v0 op_sel_hi:[0,1]
1743; GFX9-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
1744; GFX9-NEXT:    v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
1745; GFX9-NEXT:    v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
1746; GFX9-NEXT:    v_pk_lshlrev_b16 v6, 12, s2 op_sel_hi:[0,1]
1747; GFX9-NEXT:    v_pk_mul_lo_u16 v5, v1, v5
1748; GFX9-NEXT:    v_pk_mul_lo_u16 v4, v0, v4
1749; GFX9-NEXT:    v_mov_b32_e32 v0, s0
1750; GFX9-NEXT:    v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
1751; GFX9-NEXT:    v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
1752; GFX9-NEXT:    v_mov_b32_e32 v1, s1
1753; GFX9-NEXT:    v_pk_mul_lo_u16 v2, v2, v6
1754; GFX9-NEXT:    global_load_ushort v6, v[0:1], off
1755; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s7, s13
1756; GFX9-NEXT:    v_pk_lshlrev_b16 v7, 12, s2 op_sel_hi:[0,1]
1757; GFX9-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
1758; GFX9-NEXT:    v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
1759; GFX9-NEXT:    v_pk_mul_lo_u16 v3, v3, v7
1760; GFX9-NEXT:    s_waitcnt vmcnt(0)
1761; GFX9-NEXT:    v_add_u32_e32 v6, v4, v6
1762; GFX9-NEXT:    v_add_u32_sdwa v4, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1763; GFX9-NEXT:    v_add_u32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
1764; GFX9-NEXT:    v_add_u32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1765; GFX9-NEXT:    v_add_u32_e32 v4, v4, v2
1766; GFX9-NEXT:    v_add_u32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1767; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
1768; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1769; GFX9-NEXT:    global_store_short v[0:1], v2, off
1770; GFX9-NEXT:    s_endpgm
1771;
1772; GFX9-DL-LABEL: idot8_acc16_vecMul:
1773; GFX9-DL:       ; %bb.0: ; %entry
1774; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1775; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1776; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1777; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1778; GFX9-DL-NEXT:    s_load_dword s6, s[6:7], 0x0
1779; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1780; GFX9-DL-NEXT:    s_and_b32 s4, s2, 15
1781; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x40004
1782; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40008
1783; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x4000c
1784; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x40010
1785; GFX9-DL-NEXT:    s_bfe_u32 s11, s2, 0x40018
1786; GFX9-DL-NEXT:    s_lshr_b32 s12, s2, 28
1787; GFX9-DL-NEXT:    s_bfe_u32 s2, s2, 0x40014
1788; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s10, s2
1789; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v2, 12, s2 op_sel_hi:[0,1]
1790; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s11, s12
1791; GFX9-DL-NEXT:    s_bfe_u32 s7, s6, 0x40018
1792; GFX9-DL-NEXT:    s_lshr_b32 s13, s6, 28
1793; GFX9-DL-NEXT:    s_bfe_u32 s14, s6, 0x40010
1794; GFX9-DL-NEXT:    s_bfe_u32 s15, s6, 0x40014
1795; GFX9-DL-NEXT:    s_bfe_u32 s16, s6, 0x40008
1796; GFX9-DL-NEXT:    s_bfe_u32 s17, s6, 0x4000c
1797; GFX9-DL-NEXT:    s_and_b32 s18, s6, 15
1798; GFX9-DL-NEXT:    s_bfe_u32 s6, s6, 0x40004
1799; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v3, 12, s2 op_sel_hi:[0,1]
1800; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
1801; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s18, s6
1802; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v0, 12, s4 op_sel_hi:[0,1]
1803; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v4, 12, s2 op_sel_hi:[0,1]
1804; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s4, s8, s9
1805; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s16, s17
1806; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v5, 12, s2 op_sel_hi:[0,1]
1807; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v1, 12, s4 op_sel_hi:[0,1]
1808; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s14, s15
1809; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v0, 12, v0 op_sel_hi:[0,1]
1810; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
1811; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
1812; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
1813; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s2 op_sel_hi:[0,1]
1814; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v5, v1, v5
1815; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v4, v0, v4
1816; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1817; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
1818; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
1819; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1820; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v6
1821; GFX9-DL-NEXT:    global_load_ushort v6, v[0:1], off
1822; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s7, s13
1823; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v7, 12, s2 op_sel_hi:[0,1]
1824; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
1825; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
1826; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v7
1827; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
1828; GFX9-DL-NEXT:    v_add_u32_e32 v6, v4, v6
1829; GFX9-DL-NEXT:    v_add_u32_sdwa v4, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1830; GFX9-DL-NEXT:    v_add_u32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
1831; GFX9-DL-NEXT:    v_add_u32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1832; GFX9-DL-NEXT:    v_add_u32_e32 v4, v4, v2
1833; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1834; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v3
1835; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1836; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
1837; GFX9-DL-NEXT:    s_endpgm
1838;
1839; GFX10-DL-LABEL: idot8_acc16_vecMul:
1840; GFX10-DL:       ; %bb.0: ; %entry
1841; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1842; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1843; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1844; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1845; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1846; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1847; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
1848; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
1849; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
1850; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1851; GFX10-DL-NEXT:    s_and_b32 s5, s0, 15
1852; GFX10-DL-NEXT:    s_bfe_u32 s6, s0, 0x40004
1853; GFX10-DL-NEXT:    s_and_b32 s7, s1, 15
1854; GFX10-DL-NEXT:    s_bfe_u32 s8, s1, 0x40004
1855; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
1856; GFX10-DL-NEXT:    s_lshr_b32 s4, s0, 28
1857; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
1858; GFX10-DL-NEXT:    s_bfe_u32 s9, s0, 0x40010
1859; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
1860; GFX10-DL-NEXT:    s_bfe_u32 s10, s0, 0x40014
1861; GFX10-DL-NEXT:    s_bfe_u32 s6, s0, 0x40008
1862; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v3, 12, s5 op_sel_hi:[0,1]
1863; GFX10-DL-NEXT:    s_bfe_u32 s0, s0, 0x4000c
1864; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v4, 12, s7 op_sel_hi:[0,1]
1865; GFX10-DL-NEXT:    s_bfe_u32 s8, s1, 0x40008
1866; GFX10-DL-NEXT:    s_bfe_u32 s5, s1, 0x4000c
1867; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
1868; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s6, s0
1869; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
1870; GFX10-DL-NEXT:    s_bfe_u32 s6, s1, 0x40010
1871; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s5, s8, s5
1872; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x40018
1873; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
1874; GFX10-DL-NEXT:    s_bfe_u32 s0, s1, 0x40014
1875; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v4
1876; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s5 op_sel_hi:[0,1]
1877; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s5, s9, s10
1878; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v5 op_sel_hi:[0,1]
1879; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s6, s0
1880; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 28
1881; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v6 op_sel_hi:[0,1]
1882; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s5 op_sel_hi:[0,1]
1883; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v7, 12, s0 op_sel_hi:[0,1]
1884; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s2, s4
1885; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v5
1886; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s1, s7, s1
1887; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v7 op_sel_hi:[0,1]
1888; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v7, 12, s1 op_sel_hi:[0,1]
1889; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
1890; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
1891; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1892; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v3, 12, v6 op_sel_hi:[0,1]
1893; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
1894; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
1895; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v5
1896; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v7 op_sel_hi:[0,1]
1897; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1898; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v6 op_sel_hi:[0,1]
1899; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
1900; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v5
1901; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1902; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v4
1903; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1904; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
1905; GFX10-DL-NEXT:    s_endpgm
1906                                              <8 x i4> addrspace(1)* %src2,
1907                                              i16 addrspace(1)* nocapture %dst) {
1908entry:
1909  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1910  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1911
1912  %cvec1 = sext <8 x i4> %vec1 to <8 x i16>
1913  %cvec2 = sext <8 x i4> %vec2 to <8 x i16>
1914
1915  %mul = mul <8 x i16> %cvec1, %cvec2
1916  %mul0 = extractelement <8 x i16> %mul, i64 0
1917  %mul1 = extractelement <8 x i16> %mul, i64 1
1918  %mul2 = extractelement <8 x i16> %mul, i64 2
1919  %mul3 = extractelement <8 x i16> %mul, i64 3
1920  %mul4 = extractelement <8 x i16> %mul, i64 4
1921  %mul5 = extractelement <8 x i16> %mul, i64 5
1922  %mul6 = extractelement <8 x i16> %mul, i64 6
1923  %mul7 = extractelement <8 x i16> %mul, i64 7
1924
1925  %acc = load i16, i16 addrspace(1)* %dst, align 4
1926  %add1 = add i16 %mul0, %acc
1927  %add2 = add i16 %add1, %mul1
1928  %add3 = add i16 %add2, %mul2
1929  %add4 = add i16 %add3, %mul3
1930  %add5 = add i16 %add4, %mul4
1931  %add6 = add i16 %add5, %mul5
1932  %add7 = add i16 %add6, %mul6
1933  %add8 = add i16 %add7, %mul7
1934
1935  store i16 %add8, i16 addrspace(1)* %dst, align 4
1936  ret void
1937}
1938
1939; TODO: Support this pattern.
1940define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
1941; GFX7-LABEL: idot8_acc8_vecMul:
1942; GFX7:       ; %bb.0: ; %entry
1943; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
1944; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
1945; GFX7-NEXT:    s_mov_b32 s7, 0xf000
1946; GFX7-NEXT:    s_mov_b32 s6, -1
1947; GFX7-NEXT:    s_movk_i32 s0, 0xff
1948; GFX7-NEXT:    s_mov_b32 s1, 0xffff
1949; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1950; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
1951; GFX7-NEXT:    s_load_dword s2, s[8:9], 0x0
1952; GFX7-NEXT:    s_load_dword s8, s[10:11], 0x0
1953; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1954; GFX7-NEXT:    s_bfe_i32 s9, s2, 0x40000
1955; GFX7-NEXT:    s_bfe_i32 s16, s8, 0x40000
1956; GFX7-NEXT:    s_bfe_i32 s17, s8, 0x40004
1957; GFX7-NEXT:    s_bfe_i32 s18, s8, 0x40008
1958; GFX7-NEXT:    s_bfe_i32 s19, s8, 0x4000c
1959; GFX7-NEXT:    s_bfe_i32 s20, s8, 0x40010
1960; GFX7-NEXT:    s_bfe_i32 s21, s8, 0x40014
1961; GFX7-NEXT:    s_bfe_i32 s22, s8, 0x40018
1962; GFX7-NEXT:    s_ashr_i32 s8, s8, 28
1963; GFX7-NEXT:    v_mov_b32_e32 v8, s16
1964; GFX7-NEXT:    s_bfe_i32 s10, s2, 0x40004
1965; GFX7-NEXT:    v_mov_b32_e32 v7, s17
1966; GFX7-NEXT:    s_bfe_i32 s11, s2, 0x40008
1967; GFX7-NEXT:    v_mov_b32_e32 v6, s18
1968; GFX7-NEXT:    s_bfe_i32 s12, s2, 0x4000c
1969; GFX7-NEXT:    v_mov_b32_e32 v5, s19
1970; GFX7-NEXT:    s_bfe_i32 s13, s2, 0x40010
1971; GFX7-NEXT:    v_mov_b32_e32 v4, s20
1972; GFX7-NEXT:    s_bfe_i32 s14, s2, 0x40014
1973; GFX7-NEXT:    v_mov_b32_e32 v3, s21
1974; GFX7-NEXT:    s_bfe_i32 s15, s2, 0x40018
1975; GFX7-NEXT:    v_mov_b32_e32 v2, s22
1976; GFX7-NEXT:    s_ashr_i32 s2, s2, 28
1977; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1978; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s2, v1
1979; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s15, v2
1980; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s14, v3
1981; GFX7-NEXT:    v_mul_i32_i24_e32 v9, s13, v4
1982; GFX7-NEXT:    v_mul_i32_i24_e32 v5, s12, v5
1983; GFX7-NEXT:    v_mul_i32_i24_e32 v6, s11, v6
1984; GFX7-NEXT:    v_mul_i32_i24_e32 v7, s10, v7
1985; GFX7-NEXT:    v_mul_i32_i24_e32 v8, s9, v8
1986; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
1987; GFX7-NEXT:    v_and_b32_e32 v2, s0, v2
1988; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
1989; GFX7-NEXT:    v_and_b32_e32 v9, s0, v9
1990; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
1991; GFX7-NEXT:    v_and_b32_e32 v6, s0, v6
1992; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
1993; GFX7-NEXT:    v_and_b32_e32 v8, s0, v8
1994; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
1995; GFX7-NEXT:    v_or_b32_e32 v2, v9, v3
1996; GFX7-NEXT:    v_or_b32_e32 v3, v6, v5
1997; GFX7-NEXT:    v_or_b32_e32 v5, v8, v7
1998; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
1999; GFX7-NEXT:    v_and_b32_e32 v2, s1, v2
2000; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
2001; GFX7-NEXT:    v_and_b32_e32 v5, s1, v5
2002; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
2003; GFX7-NEXT:    v_or_b32_e32 v2, v5, v3
2004; GFX7-NEXT:    v_alignbit_b32 v3, v1, v2, 8
2005; GFX7-NEXT:    v_alignbit_b32 v5, v1, v2, 16
2006; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 24, v2
2007; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 8, v1
2008; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 16, v1
2009; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
2010; GFX7-NEXT:    s_waitcnt vmcnt(0)
2011; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
2012; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
2013; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
2014; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
2015; GFX7-NEXT:    v_mad_i32_i24 v0, s13, v4, v0
2016; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v7
2017; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
2018; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
2019; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
2020; GFX7-NEXT:    s_endpgm
2021;
2022; GFX8-LABEL: idot8_acc8_vecMul:
2023; GFX8:       ; %bb.0: ; %entry
2024; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2025; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2026; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2027; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2028; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2029; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
2030; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
2031; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
2032; GFX8-NEXT:    s_mov_b32 s0, 0xffff
2033; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2034; GFX8-NEXT:    s_lshr_b32 s8, s1, 4
2035; GFX8-NEXT:    s_lshr_b32 s9, s1, 12
2036; GFX8-NEXT:    s_lshr_b32 s10, s1, 8
2037; GFX8-NEXT:    s_lshr_b32 s15, s2, 4
2038; GFX8-NEXT:    s_lshr_b32 s16, s2, 12
2039; GFX8-NEXT:    s_lshr_b32 s17, s2, 8
2040; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s1
2041; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s2
2042; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s10
2043; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s9
2044; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
2045; GFX8-NEXT:    v_lshlrev_b16_e64 v12, 12, s17
2046; GFX8-NEXT:    v_lshlrev_b16_e64 v13, 12, s16
2047; GFX8-NEXT:    v_lshlrev_b16_e64 v14, 12, s15
2048; GFX8-NEXT:    s_lshr_b32 s4, s1, 20
2049; GFX8-NEXT:    s_lshr_b32 s5, s1, 16
2050; GFX8-NEXT:    s_lshr_b32 s6, s1, 28
2051; GFX8-NEXT:    s_lshr_b32 s7, s1, 24
2052; GFX8-NEXT:    s_lshr_b32 s11, s2, 20
2053; GFX8-NEXT:    s_lshr_b32 s12, s2, 16
2054; GFX8-NEXT:    s_lshr_b32 s13, s2, 28
2055; GFX8-NEXT:    s_lshr_b32 s14, s2, 24
2056; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
2057; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
2058; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
2059; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s4
2060; GFX8-NEXT:    v_lshlrev_b16_e64 v15, 12, s14
2061; GFX8-NEXT:    v_lshlrev_b16_e64 v16, 12, s13
2062; GFX8-NEXT:    v_lshlrev_b16_e64 v17, 12, s12
2063; GFX8-NEXT:    v_lshlrev_b16_e64 v18, 12, s11
2064; GFX8-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
2065; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
2066; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
2067; GFX8-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
2068; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
2069; GFX8-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
2070; GFX8-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
2071; GFX8-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
2072; GFX8-NEXT:    v_mul_u32_u24_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
2073; GFX8-NEXT:    v_mul_u32_u24_sdwa v4, v5, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
2074; GFX8-NEXT:    v_mul_u32_u24_sdwa v5, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
2075; GFX8-NEXT:    v_mul_u32_u24_sdwa v6, v7, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
2076; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
2077; GFX8-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
2078; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
2079; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
2080; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
2081; GFX8-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
2082; GFX8-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
2083; GFX8-NEXT:    v_ashrrev_i16_e32 v18, 12, v18
2084; GFX8-NEXT:    v_or_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2085; GFX8-NEXT:    v_mul_u32_u24_sdwa v7, v8, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
2086; GFX8-NEXT:    v_mul_u32_u24_sdwa v8, v9, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
2087; GFX8-NEXT:    v_mul_u32_u24_sdwa v9, v10, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
2088; GFX8-NEXT:    v_mul_u32_u24_sdwa v10, v11, v18 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
2089; GFX8-NEXT:    v_and_b32_e32 v3, s0, v3
2090; GFX8-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2091; GFX8-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2092; GFX8-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2093; GFX8-NEXT:    v_and_b32_e32 v5, s0, v9
2094; GFX8-NEXT:    v_or_b32_e32 v4, v3, v4
2095; GFX8-NEXT:    v_or_b32_e32 v6, v5, v7
2096; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 8, v4
2097; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 8, v6
2098; GFX8-NEXT:    s_waitcnt vmcnt(0)
2099; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
2100; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v7, v2
2101; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
2102; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
2103; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
2104; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v8, v2
2105; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
2106; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
2107; GFX8-NEXT:    flat_store_byte v[0:1], v2
2108; GFX8-NEXT:    s_endpgm
2109;
2110; GFX9-LABEL: idot8_acc8_vecMul:
2111; GFX9:       ; %bb.0: ; %entry
2112; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2113; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2114; GFX9-NEXT:    s_mov_b32 s2, 0xffff
2115; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2116; GFX9-NEXT:    v_mov_b32_e32 v0, s0
2117; GFX9-NEXT:    v_mov_b32_e32 v1, s1
2118; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
2119; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
2120; GFX9-NEXT:    s_load_dword s1, s[6:7], 0x0
2121; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2122; GFX9-NEXT:    s_lshr_b32 s8, s0, 4
2123; GFX9-NEXT:    s_lshr_b32 s15, s1, 4
2124; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
2125; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
2126; GFX9-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
2127; GFX9-NEXT:    v_lshlrev_b16_e64 v14, 12, s15
2128; GFX9-NEXT:    s_lshr_b32 s9, s0, 12
2129; GFX9-NEXT:    s_lshr_b32 s10, s0, 8
2130; GFX9-NEXT:    s_lshr_b32 s16, s1, 12
2131; GFX9-NEXT:    s_lshr_b32 s17, s1, 8
2132; GFX9-NEXT:    v_lshlrev_b16_e64 v5, 12, s10
2133; GFX9-NEXT:    v_lshlrev_b16_e64 v6, 12, s9
2134; GFX9-NEXT:    v_lshlrev_b16_e64 v12, 12, s17
2135; GFX9-NEXT:    v_lshlrev_b16_e64 v13, 12, s16
2136; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
2137; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
2138; GFX9-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
2139; GFX9-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
2140; GFX9-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
2141; GFX9-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
2142; GFX9-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
2143; GFX9-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
2144; GFX9-NEXT:    v_mul_lo_u16_e32 v3, v3, v4
2145; GFX9-NEXT:    v_mul_lo_u16_sdwa v7, v7, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2146; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2147; GFX9-NEXT:    s_lshr_b32 s4, s0, 20
2148; GFX9-NEXT:    s_lshr_b32 s5, s0, 16
2149; GFX9-NEXT:    s_lshr_b32 s11, s1, 20
2150; GFX9-NEXT:    s_lshr_b32 s12, s1, 16
2151; GFX9-NEXT:    v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2152; GFX9-NEXT:    v_mul_lo_u16_e32 v5, v5, v12
2153; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
2154; GFX9-NEXT:    v_lshlrev_b16_e64 v11, 12, s4
2155; GFX9-NEXT:    v_lshlrev_b16_e64 v17, 12, s12
2156; GFX9-NEXT:    v_lshlrev_b16_e64 v18, 12, s11
2157; GFX9-NEXT:    s_lshr_b32 s6, s0, 28
2158; GFX9-NEXT:    s_lshr_b32 s7, s0, 24
2159; GFX9-NEXT:    s_lshr_b32 s13, s1, 28
2160; GFX9-NEXT:    s_lshr_b32 s14, s1, 24
2161; GFX9-NEXT:    v_and_b32_e32 v3, s2, v3
2162; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2163; GFX9-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
2164; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
2165; GFX9-NEXT:    v_lshlrev_b16_e64 v15, 12, s14
2166; GFX9-NEXT:    v_lshlrev_b16_e64 v16, 12, s13
2167; GFX9-NEXT:    v_or_b32_e32 v5, v3, v5
2168; GFX9-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
2169; GFX9-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
2170; GFX9-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
2171; GFX9-NEXT:    v_ashrrev_i16_e32 v18, 12, v18
2172; GFX9-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
2173; GFX9-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
2174; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
2175; GFX9-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
2176; GFX9-NEXT:    v_mul_lo_u16_sdwa v4, v11, v18 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2177; GFX9-NEXT:    v_mul_lo_u16_e32 v10, v10, v17
2178; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 8, v5
2179; GFX9-NEXT:    v_or_b32_sdwa v4, v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2180; GFX9-NEXT:    v_mul_lo_u16_sdwa v9, v9, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2181; GFX9-NEXT:    v_mul_lo_u16_e32 v8, v8, v15
2182; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2183; GFX9-NEXT:    v_and_b32_e32 v4, s2, v4
2184; GFX9-NEXT:    v_or_b32_e32 v6, v4, v8
2185; GFX9-NEXT:    s_waitcnt vmcnt(0)
2186; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
2187; GFX9-NEXT:    v_add_u32_e32 v2, v2, v7
2188; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
2189; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2190; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
2191; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v6
2192; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
2193; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2194; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2195; GFX9-NEXT:    global_store_byte v[0:1], v2, off
2196; GFX9-NEXT:    s_endpgm
2197;
2198; GFX9-DL-LABEL: idot8_acc8_vecMul:
2199; GFX9-DL:       ; %bb.0: ; %entry
2200; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2201; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2202; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
2203; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2204; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2205; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2206; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
2207; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
2208; GFX9-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
2209; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2210; GFX9-DL-NEXT:    s_lshr_b32 s8, s0, 4
2211; GFX9-DL-NEXT:    s_lshr_b32 s15, s1, 4
2212; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
2213; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
2214; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
2215; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v14, 12, s15
2216; GFX9-DL-NEXT:    s_lshr_b32 s9, s0, 12
2217; GFX9-DL-NEXT:    s_lshr_b32 s10, s0, 8
2218; GFX9-DL-NEXT:    s_lshr_b32 s16, s1, 12
2219; GFX9-DL-NEXT:    s_lshr_b32 s17, s1, 8
2220; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s10
2221; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s9
2222; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s17
2223; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s16
2224; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
2225; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
2226; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
2227; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
2228; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
2229; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
2230; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
2231; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
2232; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v3, v3, v4
2233; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v7, v7, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2234; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2235; GFX9-DL-NEXT:    s_lshr_b32 s4, s0, 20
2236; GFX9-DL-NEXT:    s_lshr_b32 s5, s0, 16
2237; GFX9-DL-NEXT:    s_lshr_b32 s11, s1, 20
2238; GFX9-DL-NEXT:    s_lshr_b32 s12, s1, 16
2239; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2240; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v5, v5, v12
2241; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
2242; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s4
2243; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v17, 12, s12
2244; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v18, 12, s11
2245; GFX9-DL-NEXT:    s_lshr_b32 s6, s0, 28
2246; GFX9-DL-NEXT:    s_lshr_b32 s7, s0, 24
2247; GFX9-DL-NEXT:    s_lshr_b32 s13, s1, 28
2248; GFX9-DL-NEXT:    s_lshr_b32 s14, s1, 24
2249; GFX9-DL-NEXT:    v_and_b32_e32 v3, s2, v3
2250; GFX9-DL-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2251; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
2252; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
2253; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s14
2254; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v16, 12, s13
2255; GFX9-DL-NEXT:    v_or_b32_e32 v5, v3, v5
2256; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
2257; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
2258; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
2259; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v18, 12, v18
2260; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
2261; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
2262; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
2263; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
2264; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, v11, v18 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2265; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v10, v10, v17
2266; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v7, 8, v5
2267; GFX9-DL-NEXT:    v_or_b32_sdwa v4, v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2268; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v9, v9, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2269; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v8, v8, v15
2270; GFX9-DL-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2271; GFX9-DL-NEXT:    v_and_b32_e32 v4, s2, v4
2272; GFX9-DL-NEXT:    v_or_b32_e32 v6, v4, v8
2273; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2274; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
2275; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v7
2276; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
2277; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2278; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v4
2279; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v6
2280; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v3
2281; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2282; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2283; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
2284; GFX9-DL-NEXT:    s_endpgm
2285;
2286; GFX10-DL-LABEL: idot8_acc8_vecMul:
2287; GFX10-DL:       ; %bb.0: ; %entry
2288; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2289; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2290; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
2291; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2292; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2293; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2294; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2295; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
2296; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
2297; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
2298; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2299; GFX10-DL-NEXT:    s_lshr_b32 s8, s0, 4
2300; GFX10-DL-NEXT:    s_lshr_b32 s15, s1, 4
2301; GFX10-DL-NEXT:    s_lshr_b32 s9, s0, 12
2302; GFX10-DL-NEXT:    s_lshr_b32 s16, s1, 12
2303; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
2304; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
2305; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s15
2306; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
2307; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v14, 12, s16
2308; GFX10-DL-NEXT:    s_lshr_b32 s10, s0, 8
2309; GFX10-DL-NEXT:    s_lshr_b32 s17, s1, 8
2310; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s9
2311; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v7, 12, v7
2312; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v12, 12, v12
2313; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s10
2314; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v3, 12, v3
2315; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v4
2316; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s17
2317; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, v7, v12
2318; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v19, 12, v6
2319; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v14, 12, v14
2320; GFX10-DL-NEXT:    s_lshr_b32 s4, s0, 20
2321; GFX10-DL-NEXT:    s_lshr_b32 s5, s0, 16
2322; GFX10-DL-NEXT:    s_lshr_b32 s6, s0, 28
2323; GFX10-DL-NEXT:    s_lshr_b32 s7, s0, 24
2324; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v3, v3, v4
2325; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, v19, v14
2326; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v6, 8, v7
2327; GFX10-DL-NEXT:    s_lshr_b32 s11, s1, 20
2328; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v12, 12, v13
2329; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 12, v5
2330; GFX10-DL-NEXT:    s_lshr_b32 s12, s1, 16
2331; GFX10-DL-NEXT:    v_or_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2332; GFX10-DL-NEXT:    s_lshr_b32 s13, s1, 28
2333; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
2334; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
2335; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
2336; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s4
2337; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s11
2338; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, v5, v12
2339; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 8, v4
2340; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s12
2341; GFX10-DL-NEXT:    s_lshr_b32 s14, s1, 24
2342; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v6, 12, v8
2343; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v8, 12, v9
2344; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v9, 12, v10
2345; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2346; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
2347; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v16, 12, s13
2348; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 12, v11
2349; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v10, 12, v13
2350; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s14
2351; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v7, 12, v7
2352; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v11, 12, v16
2353; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v4
2354; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, v5, v10
2355; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v12, 12, v15
2356; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v10, v9, v7
2357; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v8, v8, v11
2358; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v9, 8, v4
2359; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
2360; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
2361; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v5
2362; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, v6, v12
2363; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v6, 8, v8
2364; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v9
2365; GFX10-DL-NEXT:    v_or_b32_sdwa v3, v10, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2366; GFX10-DL-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2367; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
2368; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
2369; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2370; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v5
2371; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
2372; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
2373; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
2374; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2375; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2376; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
2377; GFX10-DL-NEXT:    s_endpgm
2378                                             <8 x i4> addrspace(1)* %src2,
2379                                             i8 addrspace(1)* nocapture %dst) {
2380entry:
2381  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
2382  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
2383
2384  %cvec1 = sext <8 x i4> %vec1 to <8 x i8>
2385  %cvec2 = sext <8 x i4> %vec2 to <8 x i8>
2386
2387  %mul = mul <8 x i8> %cvec1, %cvec2
2388  %mul0 = extractelement <8 x i8> %mul, i64 0
2389  %mul1 = extractelement <8 x i8> %mul, i64 1
2390  %mul2 = extractelement <8 x i8> %mul, i64 2
2391  %mul3 = extractelement <8 x i8> %mul, i64 3
2392  %mul4 = extractelement <8 x i8> %mul, i64 4
2393  %mul5 = extractelement <8 x i8> %mul, i64 5
2394  %mul6 = extractelement <8 x i8> %mul, i64 6
2395  %mul7 = extractelement <8 x i8> %mul, i64 7
2396
2397  %acc = load i8, i8 addrspace(1)* %dst, align 4
2398  %add1 = add i8 %mul0, %acc
2399  %add2 = add i8 %add1, %mul1
2400  %add3 = add i8 %add2, %mul2
2401  %add4 = add i8 %add3, %mul3
2402  %add5 = add i8 %add4, %mul4
2403  %add6 = add i8 %add5, %mul5
2404  %add7 = add i8 %add6, %mul6
2405  %add8 = add i8 %add7, %mul7
2406
2407  store i8 %add8, i8 addrspace(1)* %dst, align 4
2408  ret void
2409}
2410