1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
8; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
9; RUN: llc -mtriple=amdgcn -mcpu=gfx1031 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
10
11define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
12; GFX7-LABEL: idot8_acc32:
13; GFX7:       ; %bb.0: ; %entry
14; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
15; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
16; GFX7-NEXT:    s_mov_b32 s24, SCRATCH_RSRC_DWORD0
17; GFX7-NEXT:    s_mov_b32 s25, SCRATCH_RSRC_DWORD1
18; GFX7-NEXT:    s_mov_b32 s26, -1
19; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
20; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
21; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
22; GFX7-NEXT:    s_load_dword s20, s[0:1], 0x0
23; GFX7-NEXT:    s_mov_b32 s27, 0xe8f000
24; GFX7-NEXT:    s_add_u32 s24, s24, s3
25; GFX7-NEXT:    s_addc_u32 s25, s25, 0
26; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
27; GFX7-NEXT:    s_bfe_i32 s7, s5, 0x40000
28; GFX7-NEXT:    s_bfe_i32 s6, s4, 0x40000
29; GFX7-NEXT:    s_bfe_i32 s9, s5, 0x40004
30; GFX7-NEXT:    v_mov_b32_e32 v0, s7
31; GFX7-NEXT:    v_mov_b32_e32 v1, s20
32; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v0, v1
33; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x40004
34; GFX7-NEXT:    v_mov_b32_e32 v1, s9
35; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x40008
36; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
37; GFX7-NEXT:    s_bfe_i32 s10, s4, 0x40008
38; GFX7-NEXT:    v_mov_b32_e32 v1, s11
39; GFX7-NEXT:    s_bfe_i32 s13, s5, 0x4000c
40; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
41; GFX7-NEXT:    s_bfe_i32 s12, s4, 0x4000c
42; GFX7-NEXT:    v_mov_b32_e32 v1, s13
43; GFX7-NEXT:    s_bfe_i32 s15, s5, 0x40010
44; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
45; GFX7-NEXT:    s_bfe_i32 s14, s4, 0x40010
46; GFX7-NEXT:    v_mov_b32_e32 v1, s15
47; GFX7-NEXT:    s_bfe_i32 s17, s5, 0x40014
48; GFX7-NEXT:    s_bfe_i32 s19, s5, 0x40018
49; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
50; GFX7-NEXT:    s_bfe_i32 s16, s4, 0x40014
51; GFX7-NEXT:    v_mov_b32_e32 v1, s17
52; GFX7-NEXT:    s_bfe_i32 s18, s4, 0x40018
53; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
54; GFX7-NEXT:    v_mov_b32_e32 v1, s19
55; GFX7-NEXT:    s_ashr_i32 s5, s5, 28
56; GFX7-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
57; GFX7-NEXT:    s_ashr_i32 s4, s4, 28
58; GFX7-NEXT:    v_mov_b32_e32 v1, s5
59; GFX7-NEXT:    s_mov_b32 s3, 0xf000
60; GFX7-NEXT:    s_mov_b32 s2, -1
61; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
62; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
63; GFX7-NEXT:    s_endpgm
64;
65; GFX8-LABEL: idot8_acc32:
66; GFX8:       ; %bb.0: ; %entry
67; GFX8-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
68; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
69; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
70; GFX8-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
71; GFX8-NEXT:    s_mov_b32 s22, -1
72; GFX8-NEXT:    s_mov_b32 s23, 0xe80000
73; GFX8-NEXT:    s_add_u32 s20, s20, s3
74; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
75; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
76; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
77; GFX8-NEXT:    s_load_dword s18, s[0:1], 0x0
78; GFX8-NEXT:    s_addc_u32 s21, s21, 0
79; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
80; GFX8-NEXT:    s_bfe_i32 s4, s2, 0x40000
81; GFX8-NEXT:    s_bfe_i32 s5, s3, 0x40000
82; GFX8-NEXT:    s_bfe_i32 s7, s3, 0x40004
83; GFX8-NEXT:    v_mov_b32_e32 v0, s5
84; GFX8-NEXT:    v_mov_b32_e32 v1, s18
85; GFX8-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
86; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x40004
87; GFX8-NEXT:    v_mov_b32_e32 v1, s7
88; GFX8-NEXT:    s_bfe_i32 s9, s3, 0x40008
89; GFX8-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
90; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x40008
91; GFX8-NEXT:    v_mov_b32_e32 v1, s9
92; GFX8-NEXT:    s_bfe_i32 s11, s3, 0x4000c
93; GFX8-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
94; GFX8-NEXT:    s_bfe_i32 s10, s2, 0x4000c
95; GFX8-NEXT:    v_mov_b32_e32 v1, s11
96; GFX8-NEXT:    s_bfe_i32 s13, s3, 0x40010
97; GFX8-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
98; GFX8-NEXT:    s_bfe_i32 s12, s2, 0x40010
99; GFX8-NEXT:    v_mov_b32_e32 v1, s13
100; GFX8-NEXT:    s_bfe_i32 s15, s3, 0x40014
101; GFX8-NEXT:    s_bfe_i32 s17, s3, 0x40018
102; GFX8-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
103; GFX8-NEXT:    s_bfe_i32 s14, s2, 0x40014
104; GFX8-NEXT:    v_mov_b32_e32 v1, s15
105; GFX8-NEXT:    s_bfe_i32 s16, s2, 0x40018
106; GFX8-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
107; GFX8-NEXT:    v_mov_b32_e32 v1, s17
108; GFX8-NEXT:    s_ashr_i32 s3, s3, 28
109; GFX8-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
110; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
111; GFX8-NEXT:    v_mov_b32_e32 v1, s3
112; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
113; GFX8-NEXT:    v_mov_b32_e32 v0, s0
114; GFX8-NEXT:    v_mov_b32_e32 v1, s1
115; GFX8-NEXT:    flat_store_dword v[0:1], v2
116; GFX8-NEXT:    s_endpgm
117;
118; GFX9-LABEL: idot8_acc32:
119; GFX9:       ; %bb.0: ; %entry
120; GFX9-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
121; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
122; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
123; GFX9-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
124; GFX9-NEXT:    s_mov_b32 s22, -1
125; GFX9-NEXT:    s_mov_b32 s23, 0xe00000
126; GFX9-NEXT:    s_add_u32 s20, s20, s3
127; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
128; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
129; GFX9-NEXT:    s_load_dword s3, s[6:7], 0x0
130; GFX9-NEXT:    s_load_dword s18, s[0:1], 0x0
131; GFX9-NEXT:    s_addc_u32 s21, s21, 0
132; GFX9-NEXT:    v_mov_b32_e32 v0, 0
133; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
134; GFX9-NEXT:    s_bfe_i32 s4, s2, 0x40000
135; GFX9-NEXT:    s_bfe_i32 s5, s3, 0x40000
136; GFX9-NEXT:    s_bfe_i32 s7, s3, 0x40004
137; GFX9-NEXT:    v_mov_b32_e32 v1, s5
138; GFX9-NEXT:    v_mov_b32_e32 v2, s18
139; GFX9-NEXT:    v_mad_i32_i24 v1, s4, v1, v2
140; GFX9-NEXT:    s_bfe_i32 s6, s2, 0x40004
141; GFX9-NEXT:    v_mov_b32_e32 v2, s7
142; GFX9-NEXT:    s_bfe_i32 s9, s3, 0x40008
143; GFX9-NEXT:    v_mad_i32_i24 v1, s6, v2, v1
144; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x40008
145; GFX9-NEXT:    v_mov_b32_e32 v2, s9
146; GFX9-NEXT:    s_bfe_i32 s11, s3, 0x4000c
147; GFX9-NEXT:    v_mad_i32_i24 v1, s8, v2, v1
148; GFX9-NEXT:    s_bfe_i32 s10, s2, 0x4000c
149; GFX9-NEXT:    v_mov_b32_e32 v2, s11
150; GFX9-NEXT:    s_bfe_i32 s13, s3, 0x40010
151; GFX9-NEXT:    v_mad_i32_i24 v1, s10, v2, v1
152; GFX9-NEXT:    s_bfe_i32 s12, s2, 0x40010
153; GFX9-NEXT:    v_mov_b32_e32 v2, s13
154; GFX9-NEXT:    s_bfe_i32 s15, s3, 0x40014
155; GFX9-NEXT:    s_bfe_i32 s17, s3, 0x40018
156; GFX9-NEXT:    v_mad_i32_i24 v1, s12, v2, v1
157; GFX9-NEXT:    s_bfe_i32 s14, s2, 0x40014
158; GFX9-NEXT:    v_mov_b32_e32 v2, s15
159; GFX9-NEXT:    s_bfe_i32 s16, s2, 0x40018
160; GFX9-NEXT:    v_mad_i32_i24 v1, s14, v2, v1
161; GFX9-NEXT:    v_mov_b32_e32 v2, s17
162; GFX9-NEXT:    s_ashr_i32 s3, s3, 28
163; GFX9-NEXT:    v_mad_i32_i24 v1, s16, v2, v1
164; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
165; GFX9-NEXT:    v_mov_b32_e32 v2, s3
166; GFX9-NEXT:    v_mad_i32_i24 v1, s2, v2, v1
167; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
168; GFX9-NEXT:    s_endpgm
169;
170; GFX9-DL-LABEL: idot8_acc32:
171; GFX9-DL:       ; %bb.0: ; %entry
172; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
173; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
174; GFX9-DL-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
175; GFX9-DL-NEXT:    s_mov_b32 s9, SCRATCH_RSRC_DWORD1
176; GFX9-DL-NEXT:    s_mov_b32 s10, -1
177; GFX9-DL-NEXT:    s_mov_b32 s11, 0xe00000
178; GFX9-DL-NEXT:    s_add_u32 s8, s8, s3
179; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
180; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
181; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
182; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
183; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
184; GFX9-DL-NEXT:    s_addc_u32 s9, s9, 0
185; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
186; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
187; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
188; GFX9-DL-NEXT:    v_dot8_i32_i4 v1, s4, v1, v2
189; GFX9-DL-NEXT:    global_store_dword v0, v1, s[0:1]
190; GFX9-DL-NEXT:    s_endpgm
191;
192; GFX10-DL-LABEL: idot8_acc32:
193; GFX10-DL:       ; %bb.0: ; %entry
194; GFX10-DL-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
195; GFX10-DL-NEXT:    s_mov_b32 s9, SCRATCH_RSRC_DWORD1
196; GFX10-DL-NEXT:    s_mov_b32 s10, -1
197; GFX10-DL-NEXT:    s_mov_b32 s11, 0x31c16000
198; GFX10-DL-NEXT:    s_add_u32 s8, s8, s3
199; GFX10-DL-NEXT:    s_clause 0x1
200; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
201; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
202; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
203; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
204; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
205; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
206; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
207; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
208; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
209; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
210; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
211; GFX10-DL-NEXT:    v_dot8_i32_i4 v0, s0, s1, v0
212; GFX10-DL-NEXT:    global_store_dword v1, v0, s[4:5]
213; GFX10-DL-NEXT:    s_endpgm
214                                       <8 x i4> addrspace(1)* %src2,
215                                       i32 addrspace(1)* nocapture %dst) {
216entry:
217  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
218  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
219
220  %v1e0 = extractelement <8 x i4> %vec1, i64 0
221  %cv1e0 = sext i4 %v1e0 to i32
222  %v2e0 = extractelement <8 x i4> %vec2, i64 0
223  %cv2e0 = sext i4 %v2e0 to i32
224  %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
225
226  %v1e1 = extractelement <8 x i4> %vec1, i64 1
227  %cv1e1 = sext i4 %v1e1 to i32
228  %v2e1 = extractelement <8 x i4> %vec2, i64 1
229  %cv2e1 = sext i4 %v2e1 to i32
230  %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
231
232  %v1e2 = extractelement <8 x i4> %vec1, i64 2
233  %cv1e2 = sext i4 %v1e2 to i32
234  %v2e2 = extractelement <8 x i4> %vec2, i64 2
235  %cv2e2 = sext i4 %v2e2 to i32
236  %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
237
238  %v1e3 = extractelement <8 x i4> %vec1, i64 3
239  %cv1e3 = sext i4 %v1e3 to i32
240  %v2e3 = extractelement <8 x i4> %vec2, i64 3
241  %cv2e3 = sext i4 %v2e3 to i32
242  %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
243
244  %v1e4 = extractelement <8 x i4> %vec1, i64 4
245  %cv1e4 = sext i4 %v1e4 to i32
246  %v2e4 = extractelement <8 x i4> %vec2, i64 4
247  %cv2e4 = sext i4 %v2e4 to i32
248  %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
249
250  %v1e5 = extractelement <8 x i4> %vec1, i64 5
251  %cv1e5 = sext i4 %v1e5 to i32
252  %v2e5 = extractelement <8 x i4> %vec2, i64 5
253  %cv2e5 = sext i4 %v2e5 to i32
254  %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
255
256  %v1e6 = extractelement <8 x i4> %vec1, i64 6
257  %cv1e6 = sext i4 %v1e6 to i32
258  %v2e6 = extractelement <8 x i4> %vec2, i64 6
259  %cv2e6 = sext i4 %v2e6 to i32
260  %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
261
262  %v1e7 = extractelement <8 x i4> %vec1, i64 7
263  %cv1e7 = sext i4 %v1e7 to i32
264  %v2e7 = extractelement <8 x i4> %vec2, i64 7
265  %cv2e7 = sext i4 %v2e7 to i32
266  %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
267
268  %acc = load i32, i32 addrspace(1)* %dst, align 4
269  %add1 = add i32 %mul0, %acc
270  %add2 = add i32 %add1, %mul1
271  %add3 = add i32 %add2, %mul2
272  %add4 = add i32 %add3, %mul3
273  %add5 = add i32 %add4, %mul4
274  %add6 = add i32 %add5, %mul5
275  %add7 = add i32 %add6, %mul6
276  %add8 = add i32 %add7, %mul7
277
278  store i32 %add8, i32 addrspace(1)* %dst, align 4
279  ret void
280}
281
282; TODO: Once the unnecessary zero extentions of the elements are removed;
283; pattern recognizer will kick in.
284define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
285; GFX7-LABEL: idot8_acc16:
286; GFX7:       ; %bb.0: ; %entry
287; GFX7-NEXT:    s_mov_b32 s24, SCRATCH_RSRC_DWORD0
288; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
289; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
290; GFX7-NEXT:    s_mov_b32 s25, SCRATCH_RSRC_DWORD1
291; GFX7-NEXT:    s_mov_b32 s26, -1
292; GFX7-NEXT:    s_mov_b32 s27, 0xe8f000
293; GFX7-NEXT:    s_add_u32 s24, s24, s3
294; GFX7-NEXT:    s_mov_b32 s3, 0xf000
295; GFX7-NEXT:    s_mov_b32 s2, -1
296; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
297; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
298; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
299; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
300; GFX7-NEXT:    s_addc_u32 s25, s25, 0
301; GFX7-NEXT:    s_mov_b32 s8, 0xffff
302; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
303; GFX7-NEXT:    s_bfe_i32 s6, s4, 0x40000
304; GFX7-NEXT:    s_bfe_i32 s7, s5, 0x40000
305; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x40004
306; GFX7-NEXT:    s_and_b32 s7, s7, s8
307; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x40004
308; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x40008
309; GFX7-NEXT:    s_and_b32 s10, s10, s8
310; GFX7-NEXT:    s_and_b32 s6, s6, s8
311; GFX7-NEXT:    v_mov_b32_e32 v1, s7
312; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x40008
313; GFX7-NEXT:    s_bfe_i32 s14, s5, 0x4000c
314; GFX7-NEXT:    s_and_b32 s12, s12, s8
315; GFX7-NEXT:    s_and_b32 s9, s9, s8
316; GFX7-NEXT:    v_mov_b32_e32 v2, s10
317; GFX7-NEXT:    s_bfe_i32 s13, s4, 0x4000c
318; GFX7-NEXT:    s_bfe_i32 s16, s5, 0x40010
319; GFX7-NEXT:    s_and_b32 s14, s14, s8
320; GFX7-NEXT:    s_and_b32 s11, s11, s8
321; GFX7-NEXT:    v_mov_b32_e32 v3, s12
322; GFX7-NEXT:    s_bfe_i32 s15, s4, 0x40010
323; GFX7-NEXT:    s_bfe_i32 s18, s5, 0x40014
324; GFX7-NEXT:    s_and_b32 s16, s16, s8
325; GFX7-NEXT:    s_and_b32 s13, s13, s8
326; GFX7-NEXT:    v_mov_b32_e32 v4, s14
327; GFX7-NEXT:    s_bfe_i32 s20, s5, 0x40018
328; GFX7-NEXT:    s_bfe_i32 s17, s4, 0x40014
329; GFX7-NEXT:    s_and_b32 s18, s18, s8
330; GFX7-NEXT:    s_and_b32 s15, s15, s8
331; GFX7-NEXT:    v_mov_b32_e32 v5, s16
332; GFX7-NEXT:    s_bfe_i32 s19, s4, 0x40018
333; GFX7-NEXT:    s_ashr_i32 s5, s5, 28
334; GFX7-NEXT:    s_and_b32 s20, s20, s8
335; GFX7-NEXT:    s_and_b32 s17, s17, s8
336; GFX7-NEXT:    v_mov_b32_e32 v6, s18
337; GFX7-NEXT:    s_ashr_i32 s4, s4, 28
338; GFX7-NEXT:    s_and_b32 s19, s19, s8
339; GFX7-NEXT:    s_and_b32 s5, s5, s8
340; GFX7-NEXT:    v_mov_b32_e32 v7, s20
341; GFX7-NEXT:    s_and_b32 s4, s4, s8
342; GFX7-NEXT:    s_waitcnt vmcnt(0)
343; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
344; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
345; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
346; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v4, v0
347; GFX7-NEXT:    v_mad_u32_u24 v0, s15, v5, v0
348; GFX7-NEXT:    v_mad_u32_u24 v0, s17, v6, v0
349; GFX7-NEXT:    v_mad_u32_u24 v0, s19, v7, v0
350; GFX7-NEXT:    v_mov_b32_e32 v1, s5
351; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
352; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
353; GFX7-NEXT:    s_endpgm
354;
355; GFX8-LABEL: idot8_acc16:
356; GFX8:       ; %bb.0: ; %entry
357; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
358; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
359; GFX8-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
360; GFX8-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
361; GFX8-NEXT:    s_mov_b32 s18, -1
362; GFX8-NEXT:    s_mov_b32 s19, 0xe80000
363; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
364; GFX8-NEXT:    v_mov_b32_e32 v0, s0
365; GFX8-NEXT:    v_mov_b32_e32 v1, s1
366; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
367; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
368; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
369; GFX8-NEXT:    s_add_u32 s16, s16, s3
370; GFX8-NEXT:    s_addc_u32 s17, s17, 0
371; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
372; GFX8-NEXT:    s_bfe_i32 s4, s0, 0x40000
373; GFX8-NEXT:    s_bfe_i32 s5, s1, 0x40000
374; GFX8-NEXT:    s_bfe_i32 s7, s1, 0x40004
375; GFX8-NEXT:    s_bfe_i32 s9, s1, 0x40008
376; GFX8-NEXT:    v_mov_b32_e32 v6, s5
377; GFX8-NEXT:    s_lshr_b32 s2, s0, 12
378; GFX8-NEXT:    s_lshr_b32 s3, s1, 12
379; GFX8-NEXT:    s_bfe_i32 s6, s0, 0x40004
380; GFX8-NEXT:    s_bfe_i32 s8, s0, 0x40008
381; GFX8-NEXT:    v_mov_b32_e32 v3, s9
382; GFX8-NEXT:    v_mov_b32_e32 v7, s7
383; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s2
384; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s3
385; GFX8-NEXT:    v_mul_i32_i24_e32 v3, s8, v3
386; GFX8-NEXT:    s_bfe_i32 s11, s1, 0x40010
387; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
388; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
389; GFX8-NEXT:    s_bfe_i32 s13, s1, 0x40014
390; GFX8-NEXT:    s_bfe_i32 s10, s0, 0x40010
391; GFX8-NEXT:    v_mov_b32_e32 v8, s11
392; GFX8-NEXT:    s_bfe_i32 s15, s1, 0x40018
393; GFX8-NEXT:    s_bfe_i32 s12, s0, 0x40014
394; GFX8-NEXT:    v_mov_b32_e32 v9, s13
395; GFX8-NEXT:    s_bfe_i32 s14, s0, 0x40018
396; GFX8-NEXT:    s_ashr_i32 s1, s1, 28
397; GFX8-NEXT:    v_mov_b32_e32 v10, s15
398; GFX8-NEXT:    s_ashr_i32 s0, s0, 28
399; GFX8-NEXT:    s_waitcnt vmcnt(0)
400; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v6, v2
401; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v7, v2
402; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
403; GFX8-NEXT:    v_mad_u32_u24 v2, v4, v5, v2
404; GFX8-NEXT:    v_mad_i32_i24 v2, s10, v8, v2
405; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v9, v2
406; GFX8-NEXT:    v_mad_i32_i24 v2, s14, v10, v2
407; GFX8-NEXT:    v_mov_b32_e32 v3, s1
408; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
409; GFX8-NEXT:    flat_store_short v[0:1], v2
410; GFX8-NEXT:    s_endpgm
411;
412; GFX9-LABEL: idot8_acc16:
413; GFX9:       ; %bb.0: ; %entry
414; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
415; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
416; GFX9-NEXT:    v_mov_b32_e32 v0, 0
417; GFX9-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
418; GFX9-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
419; GFX9-NEXT:    s_mov_b32 s22, -1
420; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
421; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1]
422; GFX9-NEXT:    s_mov_b32 s23, 0xe00000
423; GFX9-NEXT:    s_add_u32 s20, s20, s3
424; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
425; GFX9-NEXT:    s_load_dword s3, s[6:7], 0x0
426; GFX9-NEXT:    s_addc_u32 s21, s21, 0
427; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
428; GFX9-NEXT:    s_bfe_i32 s6, s2, 0x40000
429; GFX9-NEXT:    s_bfe_i32 s7, s3, 0x40000
430; GFX9-NEXT:    s_bfe_i32 s9, s3, 0x40004
431; GFX9-NEXT:    s_bfe_i32 s11, s3, 0x40008
432; GFX9-NEXT:    v_mov_b32_e32 v5, s7
433; GFX9-NEXT:    s_lshr_b32 s4, s2, 12
434; GFX9-NEXT:    s_lshr_b32 s5, s3, 12
435; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x40004
436; GFX9-NEXT:    s_bfe_i32 s10, s2, 0x40008
437; GFX9-NEXT:    v_mov_b32_e32 v2, s11
438; GFX9-NEXT:    v_mov_b32_e32 v6, s9
439; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 12, s4
440; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s5
441; GFX9-NEXT:    v_mul_i32_i24_e32 v2, s10, v2
442; GFX9-NEXT:    s_bfe_i32 s13, s3, 0x40010
443; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
444; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
445; GFX9-NEXT:    s_bfe_i32 s15, s3, 0x40014
446; GFX9-NEXT:    s_bfe_i32 s12, s2, 0x40010
447; GFX9-NEXT:    v_mov_b32_e32 v7, s13
448; GFX9-NEXT:    s_bfe_i32 s17, s3, 0x40018
449; GFX9-NEXT:    s_bfe_i32 s14, s2, 0x40014
450; GFX9-NEXT:    v_mov_b32_e32 v8, s15
451; GFX9-NEXT:    s_bfe_i32 s16, s2, 0x40018
452; GFX9-NEXT:    s_ashr_i32 s3, s3, 28
453; GFX9-NEXT:    v_mov_b32_e32 v9, s17
454; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
455; GFX9-NEXT:    s_waitcnt vmcnt(0)
456; GFX9-NEXT:    v_mad_i32_i24 v1, s6, v5, v1
457; GFX9-NEXT:    v_mad_i32_i24 v1, s8, v6, v1
458; GFX9-NEXT:    v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
459; GFX9-NEXT:    v_mad_u32_u24 v1, v3, v4, v1
460; GFX9-NEXT:    v_mad_i32_i24 v1, s12, v7, v1
461; GFX9-NEXT:    v_mad_i32_i24 v1, s14, v8, v1
462; GFX9-NEXT:    v_mad_i32_i24 v1, s16, v9, v1
463; GFX9-NEXT:    v_mov_b32_e32 v2, s3
464; GFX9-NEXT:    v_mad_i32_i24 v1, s2, v2, v1
465; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
466; GFX9-NEXT:    s_endpgm
467;
468; GFX9-DL-LABEL: idot8_acc16:
469; GFX9-DL:       ; %bb.0: ; %entry
470; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
471; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
472; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
473; GFX9-DL-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
474; GFX9-DL-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
475; GFX9-DL-NEXT:    s_mov_b32 s22, -1
476; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
477; GFX9-DL-NEXT:    global_load_ushort v1, v0, s[0:1]
478; GFX9-DL-NEXT:    s_mov_b32 s23, 0xe00000
479; GFX9-DL-NEXT:    s_add_u32 s20, s20, s3
480; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
481; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
482; GFX9-DL-NEXT:    s_addc_u32 s21, s21, 0
483; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
484; GFX9-DL-NEXT:    s_bfe_i32 s6, s2, 0x40000
485; GFX9-DL-NEXT:    s_bfe_i32 s7, s3, 0x40000
486; GFX9-DL-NEXT:    s_bfe_i32 s9, s3, 0x40004
487; GFX9-DL-NEXT:    s_bfe_i32 s11, s3, 0x40008
488; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s7
489; GFX9-DL-NEXT:    s_lshr_b32 s4, s2, 12
490; GFX9-DL-NEXT:    s_lshr_b32 s5, s3, 12
491; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x40004
492; GFX9-DL-NEXT:    s_bfe_i32 s10, s2, 0x40008
493; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s11
494; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s9
495; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s4
496; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s5
497; GFX9-DL-NEXT:    v_mul_i32_i24_e32 v2, s10, v2
498; GFX9-DL-NEXT:    s_bfe_i32 s13, s3, 0x40010
499; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
500; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
501; GFX9-DL-NEXT:    s_bfe_i32 s15, s3, 0x40014
502; GFX9-DL-NEXT:    s_bfe_i32 s12, s2, 0x40010
503; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s13
504; GFX9-DL-NEXT:    s_bfe_i32 s17, s3, 0x40018
505; GFX9-DL-NEXT:    s_bfe_i32 s14, s2, 0x40014
506; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s15
507; GFX9-DL-NEXT:    s_bfe_i32 s16, s2, 0x40018
508; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 28
509; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s17
510; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 28
511; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
512; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s6, v5, v1
513; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s8, v6, v1
514; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
515; GFX9-DL-NEXT:    v_mad_u32_u24 v1, v3, v4, v1
516; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s12, v7, v1
517; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s14, v8, v1
518; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s16, v9, v1
519; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
520; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s2, v2, v1
521; GFX9-DL-NEXT:    global_store_short v0, v1, s[0:1]
522; GFX9-DL-NEXT:    s_endpgm
523;
524; GFX10-DL-LABEL: idot8_acc16:
525; GFX10-DL:       ; %bb.0: ; %entry
526; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
527; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
528; GFX10-DL-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
529; GFX10-DL-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
530; GFX10-DL-NEXT:    s_mov_b32 s14, -1
531; GFX10-DL-NEXT:    s_mov_b32 s15, 0x31c16000
532; GFX10-DL-NEXT:    s_add_u32 s12, s12, s3
533; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
534; GFX10-DL-NEXT:    s_addc_u32 s13, s13, 0
535; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
536; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
537; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
538; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
539; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
540; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
541; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 12
542; GFX10-DL-NEXT:    s_lshr_b32 s3, s1, 12
543; GFX10-DL-NEXT:    s_bfe_i32 s6, s0, 0x40000
544; GFX10-DL-NEXT:    s_bfe_i32 s7, s1, 0x40000
545; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v2, 12, s2
546; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s3
547; GFX10-DL-NEXT:    s_bfe_i32 s8, s0, 0x40004
548; GFX10-DL-NEXT:    s_bfe_i32 s9, s0, 0x40008
549; GFX10-DL-NEXT:    s_bfe_i32 s10, s1, 0x40008
550; GFX10-DL-NEXT:    s_bfe_i32 s2, s1, 0x40004
551; GFX10-DL-NEXT:    v_mul_i32_i24_e64 v4, s9, s10
552; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v2, 12, v2
553; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v3, 12, v3
554; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x40010
555; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
556; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s6, s7, v1
557; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s8, s2, v1
558; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
559; GFX10-DL-NEXT:    v_and_b32_e32 v2, s2, v2
560; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
561; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
562; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40010
563; GFX10-DL-NEXT:    v_mad_u32_u24 v1, v2, v3, v1
564; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
565; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40014
566; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x40014
567; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
568; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40018
569; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x40018
570; GFX10-DL-NEXT:    s_ashr_i32 s0, s0, 28
571; GFX10-DL-NEXT:    s_ashr_i32 s1, s1, 28
572; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
573; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s0, s1, v1
574; GFX10-DL-NEXT:    global_store_short v0, v1, s[4:5]
575; GFX10-DL-NEXT:    s_endpgm
576                                       <8 x i4> addrspace(1)* %src2,
577                                       i16 addrspace(1)* nocapture %dst) {
578entry:
579  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
580  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
581
582  %v1e0 = extractelement <8 x i4> %vec1, i64 0
583  %cv1e0 = sext i4 %v1e0 to i16
584  %v2e0 = extractelement <8 x i4> %vec2, i64 0
585  %cv2e0 = sext i4 %v2e0 to i16
586  %mul0 = mul nuw nsw i16 %cv1e0, %cv2e0
587
588  %v1e1 = extractelement <8 x i4> %vec1, i64 1
589  %cv1e1 = sext i4 %v1e1 to i16
590  %v2e1 = extractelement <8 x i4> %vec2, i64 1
591  %cv2e1 = sext i4 %v2e1 to i16
592  %mul1 = mul nuw nsw i16 %cv1e1, %cv2e1
593
594  %v1e2 = extractelement <8 x i4> %vec1, i64 2
595  %cv1e2 = sext i4 %v1e2 to i16
596  %v2e2 = extractelement <8 x i4> %vec2, i64 2
597  %cv2e2 = sext i4 %v2e2 to i16
598  %mul2 = mul nuw nsw i16 %cv1e2, %cv2e2
599
600  %v1e3 = extractelement <8 x i4> %vec1, i64 3
601  %cv1e3 = sext i4 %v1e3 to i16
602  %v2e3 = extractelement <8 x i4> %vec2, i64 3
603  %cv2e3 = sext i4 %v2e3 to i16
604  %mul3 = mul nuw nsw i16 %cv1e3, %cv2e3
605
606  %v1e4 = extractelement <8 x i4> %vec1, i64 4
607  %cv1e4 = sext i4 %v1e4 to i16
608  %v2e4 = extractelement <8 x i4> %vec2, i64 4
609  %cv2e4 = sext i4 %v2e4 to i16
610  %mul4 = mul nuw nsw i16 %cv1e4, %cv2e4
611
612  %v1e5 = extractelement <8 x i4> %vec1, i64 5
613  %cv1e5 = sext i4 %v1e5 to i16
614  %v2e5 = extractelement <8 x i4> %vec2, i64 5
615  %cv2e5 = sext i4 %v2e5 to i16
616  %mul5 = mul nuw nsw i16 %cv1e5, %cv2e5
617
618  %v1e6 = extractelement <8 x i4> %vec1, i64 6
619  %cv1e6 = sext i4 %v1e6 to i16
620  %v2e6 = extractelement <8 x i4> %vec2, i64 6
621  %cv2e6 = sext i4 %v2e6 to i16
622  %mul6 = mul nuw nsw i16 %cv1e6, %cv2e6
623
624  %v1e7 = extractelement <8 x i4> %vec1, i64 7
625  %cv1e7 = sext i4 %v1e7 to i16
626  %v2e7 = extractelement <8 x i4> %vec2, i64 7
627  %cv2e7 = sext i4 %v2e7 to i16
628  %mul7 = mul nuw nsw i16 %cv1e7, %cv2e7
629
630  %acc = load i16, i16 addrspace(1)* %dst, align 4
631  %add1 = add i16 %mul0, %acc
632  %add2 = add i16 %add1, %mul1
633  %add3 = add i16 %add2, %mul2
634  %add4 = add i16 %add3, %mul3
635  %add5 = add i16 %add4, %mul4
636  %add6 = add i16 %add5, %mul5
637  %add7 = add i16 %add6, %mul6
638  %add8 = add i16 %add7, %mul7
639
640  store i16 %add8, i16 addrspace(1)* %dst, align 4
641  ret void
642}
643
644; TODO: Support this pattern.
645define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
646; GFX7-LABEL: idot8_acc8:
647; GFX7:       ; %bb.0: ; %entry
648; GFX7-NEXT:    s_mov_b32 s24, SCRATCH_RSRC_DWORD0
649; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
650; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
651; GFX7-NEXT:    s_mov_b32 s25, SCRATCH_RSRC_DWORD1
652; GFX7-NEXT:    s_mov_b32 s26, -1
653; GFX7-NEXT:    s_mov_b32 s27, 0xe8f000
654; GFX7-NEXT:    s_add_u32 s24, s24, s3
655; GFX7-NEXT:    s_mov_b32 s3, 0xf000
656; GFX7-NEXT:    s_mov_b32 s2, -1
657; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
658; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
659; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
660; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
661; GFX7-NEXT:    s_addc_u32 s25, s25, 0
662; GFX7-NEXT:    s_movk_i32 s8, 0xff
663; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
664; GFX7-NEXT:    s_bfe_i32 s6, s4, 0x40000
665; GFX7-NEXT:    s_bfe_i32 s7, s5, 0x40000
666; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x40004
667; GFX7-NEXT:    s_and_b32 s7, s7, s8
668; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x40004
669; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x40008
670; GFX7-NEXT:    s_and_b32 s10, s10, s8
671; GFX7-NEXT:    s_and_b32 s6, s6, s8
672; GFX7-NEXT:    v_mov_b32_e32 v1, s7
673; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x40008
674; GFX7-NEXT:    s_bfe_i32 s14, s5, 0x4000c
675; GFX7-NEXT:    s_and_b32 s12, s12, s8
676; GFX7-NEXT:    s_and_b32 s9, s9, s8
677; GFX7-NEXT:    v_mov_b32_e32 v2, s10
678; GFX7-NEXT:    s_bfe_i32 s13, s4, 0x4000c
679; GFX7-NEXT:    s_bfe_i32 s16, s5, 0x40010
680; GFX7-NEXT:    s_and_b32 s14, s14, s8
681; GFX7-NEXT:    s_and_b32 s11, s11, s8
682; GFX7-NEXT:    v_mov_b32_e32 v3, s12
683; GFX7-NEXT:    s_bfe_i32 s15, s4, 0x40010
684; GFX7-NEXT:    s_bfe_i32 s18, s5, 0x40014
685; GFX7-NEXT:    s_and_b32 s16, s16, s8
686; GFX7-NEXT:    s_and_b32 s13, s13, s8
687; GFX7-NEXT:    v_mov_b32_e32 v4, s14
688; GFX7-NEXT:    s_bfe_i32 s20, s5, 0x40018
689; GFX7-NEXT:    s_bfe_i32 s17, s4, 0x40014
690; GFX7-NEXT:    s_and_b32 s18, s18, s8
691; GFX7-NEXT:    s_and_b32 s15, s15, s8
692; GFX7-NEXT:    v_mov_b32_e32 v5, s16
693; GFX7-NEXT:    s_bfe_i32 s19, s4, 0x40018
694; GFX7-NEXT:    s_ashr_i32 s5, s5, 28
695; GFX7-NEXT:    s_and_b32 s20, s20, s8
696; GFX7-NEXT:    s_and_b32 s17, s17, s8
697; GFX7-NEXT:    v_mov_b32_e32 v6, s18
698; GFX7-NEXT:    s_ashr_i32 s4, s4, 28
699; GFX7-NEXT:    s_and_b32 s19, s19, s8
700; GFX7-NEXT:    s_and_b32 s5, s5, s8
701; GFX7-NEXT:    v_mov_b32_e32 v7, s20
702; GFX7-NEXT:    s_and_b32 s4, s4, s8
703; GFX7-NEXT:    s_waitcnt vmcnt(0)
704; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
705; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
706; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
707; GFX7-NEXT:    v_mad_u32_u24 v0, s13, v4, v0
708; GFX7-NEXT:    v_mad_u32_u24 v0, s15, v5, v0
709; GFX7-NEXT:    v_mad_u32_u24 v0, s17, v6, v0
710; GFX7-NEXT:    v_mad_u32_u24 v0, s19, v7, v0
711; GFX7-NEXT:    v_mov_b32_e32 v1, s5
712; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
713; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
714; GFX7-NEXT:    s_endpgm
715;
716; GFX8-LABEL: idot8_acc8:
717; GFX8:       ; %bb.0: ; %entry
718; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
719; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
720; GFX8-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
721; GFX8-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
722; GFX8-NEXT:    s_mov_b32 s22, -1
723; GFX8-NEXT:    s_mov_b32 s23, 0xe80000
724; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
725; GFX8-NEXT:    v_mov_b32_e32 v0, s0
726; GFX8-NEXT:    v_mov_b32_e32 v1, s1
727; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
728; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
729; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
730; GFX8-NEXT:    s_add_u32 s20, s20, s3
731; GFX8-NEXT:    s_addc_u32 s21, s21, 0
732; GFX8-NEXT:    s_movk_i32 s0, 0xff
733; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
734; GFX8-NEXT:    s_lshr_b32 s3, s1, 12
735; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x40000
736; GFX8-NEXT:    s_lshr_b32 s4, s2, 12
737; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x40004
738; GFX8-NEXT:    s_bfe_i32 s10, s2, 0x40008
739; GFX8-NEXT:    s_bfe_i32 s5, s1, 0x40000
740; GFX8-NEXT:    v_mov_b32_e32 v6, s6
741; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s3
742; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s4
743; GFX8-NEXT:    s_bfe_i32 s7, s1, 0x40004
744; GFX8-NEXT:    s_bfe_i32 s9, s1, 0x40008
745; GFX8-NEXT:    v_mov_b32_e32 v3, s10
746; GFX8-NEXT:    v_mov_b32_e32 v7, s8
747; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
748; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
749; GFX8-NEXT:    v_mul_i32_i24_e32 v3, s9, v3
750; GFX8-NEXT:    s_bfe_i32 s12, s2, 0x40010
751; GFX8-NEXT:    v_and_b32_e32 v4, s0, v4
752; GFX8-NEXT:    v_and_b32_e32 v5, s0, v5
753; GFX8-NEXT:    s_bfe_i32 s14, s2, 0x40014
754; GFX8-NEXT:    s_bfe_i32 s11, s1, 0x40010
755; GFX8-NEXT:    v_mov_b32_e32 v8, s12
756; GFX8-NEXT:    s_bfe_i32 s16, s2, 0x40018
757; GFX8-NEXT:    s_bfe_i32 s13, s1, 0x40014
758; GFX8-NEXT:    v_mov_b32_e32 v9, s14
759; GFX8-NEXT:    s_bfe_i32 s15, s1, 0x40018
760; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
761; GFX8-NEXT:    v_mov_b32_e32 v10, s16
762; GFX8-NEXT:    s_ashr_i32 s1, s1, 28
763; GFX8-NEXT:    s_waitcnt vmcnt(0)
764; GFX8-NEXT:    v_mad_i32_i24 v2, s5, v6, v2
765; GFX8-NEXT:    v_mad_i32_i24 v2, s7, v7, v2
766; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
767; GFX8-NEXT:    v_mad_u32_u24 v2, v4, v5, v2
768; GFX8-NEXT:    v_mad_i32_i24 v2, s11, v8, v2
769; GFX8-NEXT:    v_mad_i32_i24 v2, s13, v9, v2
770; GFX8-NEXT:    v_mad_i32_i24 v2, s15, v10, v2
771; GFX8-NEXT:    v_mov_b32_e32 v3, s2
772; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
773; GFX8-NEXT:    flat_store_byte v[0:1], v2
774; GFX8-NEXT:    s_endpgm
775;
776; GFX9-LABEL: idot8_acc8:
777; GFX9:       ; %bb.0: ; %entry
778; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
779; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
780; GFX9-NEXT:    v_mov_b32_e32 v0, 0
781; GFX9-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
782; GFX9-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
783; GFX9-NEXT:    s_mov_b32 s22, -1
784; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
785; GFX9-NEXT:    global_load_ubyte v1, v0, s[0:1]
786; GFX9-NEXT:    s_mov_b32 s23, 0xe00000
787; GFX9-NEXT:    s_add_u32 s20, s20, s3
788; GFX9-NEXT:    s_load_dword s3, s[4:5], 0x0
789; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
790; GFX9-NEXT:    s_addc_u32 s21, s21, 0
791; GFX9-NEXT:    s_movk_i32 s2, 0xff
792; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
793; GFX9-NEXT:    s_lshr_b32 s5, s3, 12
794; GFX9-NEXT:    s_bfe_i32 s8, s4, 0x40000
795; GFX9-NEXT:    s_lshr_b32 s6, s4, 12
796; GFX9-NEXT:    s_bfe_i32 s10, s4, 0x40004
797; GFX9-NEXT:    s_bfe_i32 s12, s4, 0x40008
798; GFX9-NEXT:    s_bfe_i32 s7, s3, 0x40000
799; GFX9-NEXT:    v_mov_b32_e32 v5, s8
800; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 12, s5
801; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s6
802; GFX9-NEXT:    s_bfe_i32 s9, s3, 0x40004
803; GFX9-NEXT:    s_bfe_i32 s11, s3, 0x40008
804; GFX9-NEXT:    v_mov_b32_e32 v2, s12
805; GFX9-NEXT:    v_mov_b32_e32 v6, s10
806; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
807; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
808; GFX9-NEXT:    v_mul_i32_i24_e32 v2, s11, v2
809; GFX9-NEXT:    s_bfe_i32 s14, s4, 0x40010
810; GFX9-NEXT:    v_and_b32_e32 v3, s2, v3
811; GFX9-NEXT:    v_and_b32_e32 v4, s2, v4
812; GFX9-NEXT:    s_bfe_i32 s16, s4, 0x40014
813; GFX9-NEXT:    s_bfe_i32 s13, s3, 0x40010
814; GFX9-NEXT:    v_mov_b32_e32 v7, s14
815; GFX9-NEXT:    s_bfe_i32 s18, s4, 0x40018
816; GFX9-NEXT:    s_bfe_i32 s15, s3, 0x40014
817; GFX9-NEXT:    v_mov_b32_e32 v8, s16
818; GFX9-NEXT:    s_bfe_i32 s17, s3, 0x40018
819; GFX9-NEXT:    s_ashr_i32 s4, s4, 28
820; GFX9-NEXT:    v_mov_b32_e32 v9, s18
821; GFX9-NEXT:    s_ashr_i32 s3, s3, 28
822; GFX9-NEXT:    s_waitcnt vmcnt(0)
823; GFX9-NEXT:    v_mad_i32_i24 v1, s7, v5, v1
824; GFX9-NEXT:    v_mad_i32_i24 v1, s9, v6, v1
825; GFX9-NEXT:    v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
826; GFX9-NEXT:    v_mad_u32_u24 v1, v3, v4, v1
827; GFX9-NEXT:    v_mad_i32_i24 v1, s13, v7, v1
828; GFX9-NEXT:    v_mad_i32_i24 v1, s15, v8, v1
829; GFX9-NEXT:    v_mad_i32_i24 v1, s17, v9, v1
830; GFX9-NEXT:    v_mov_b32_e32 v2, s4
831; GFX9-NEXT:    v_mad_i32_i24 v1, s3, v2, v1
832; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
833; GFX9-NEXT:    s_endpgm
834;
835; GFX9-DL-LABEL: idot8_acc8:
836; GFX9-DL:       ; %bb.0: ; %entry
837; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
838; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
839; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
840; GFX9-DL-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
841; GFX9-DL-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
842; GFX9-DL-NEXT:    s_mov_b32 s22, -1
843; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
844; GFX9-DL-NEXT:    global_load_ubyte v1, v0, s[0:1]
845; GFX9-DL-NEXT:    s_mov_b32 s23, 0xe00000
846; GFX9-DL-NEXT:    s_add_u32 s20, s20, s3
847; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
848; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
849; GFX9-DL-NEXT:    s_addc_u32 s21, s21, 0
850; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
851; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
852; GFX9-DL-NEXT:    s_lshr_b32 s5, s3, 12
853; GFX9-DL-NEXT:    s_bfe_i32 s8, s4, 0x40000
854; GFX9-DL-NEXT:    s_lshr_b32 s6, s4, 12
855; GFX9-DL-NEXT:    s_bfe_i32 s10, s4, 0x40004
856; GFX9-DL-NEXT:    s_bfe_i32 s12, s4, 0x40008
857; GFX9-DL-NEXT:    s_bfe_i32 s7, s3, 0x40000
858; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s8
859; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s5
860; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s6
861; GFX9-DL-NEXT:    s_bfe_i32 s9, s3, 0x40004
862; GFX9-DL-NEXT:    s_bfe_i32 s11, s3, 0x40008
863; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s12
864; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s10
865; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
866; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
867; GFX9-DL-NEXT:    v_mul_i32_i24_e32 v2, s11, v2
868; GFX9-DL-NEXT:    s_bfe_i32 s14, s4, 0x40010
869; GFX9-DL-NEXT:    v_and_b32_e32 v3, s2, v3
870; GFX9-DL-NEXT:    v_and_b32_e32 v4, s2, v4
871; GFX9-DL-NEXT:    s_bfe_i32 s16, s4, 0x40014
872; GFX9-DL-NEXT:    s_bfe_i32 s13, s3, 0x40010
873; GFX9-DL-NEXT:    v_mov_b32_e32 v7, s14
874; GFX9-DL-NEXT:    s_bfe_i32 s18, s4, 0x40018
875; GFX9-DL-NEXT:    s_bfe_i32 s15, s3, 0x40014
876; GFX9-DL-NEXT:    v_mov_b32_e32 v8, s16
877; GFX9-DL-NEXT:    s_bfe_i32 s17, s3, 0x40018
878; GFX9-DL-NEXT:    s_ashr_i32 s4, s4, 28
879; GFX9-DL-NEXT:    v_mov_b32_e32 v9, s18
880; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 28
881; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
882; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s7, v5, v1
883; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s9, v6, v1
884; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
885; GFX9-DL-NEXT:    v_mad_u32_u24 v1, v3, v4, v1
886; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s13, v7, v1
887; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s15, v8, v1
888; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s17, v9, v1
889; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
890; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s3, v2, v1
891; GFX9-DL-NEXT:    global_store_byte v0, v1, s[0:1]
892; GFX9-DL-NEXT:    s_endpgm
893;
894; GFX10-DL-LABEL: idot8_acc8:
895; GFX10-DL:       ; %bb.0: ; %entry
896; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
897; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
898; GFX10-DL-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
899; GFX10-DL-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
900; GFX10-DL-NEXT:    s_mov_b32 s14, -1
901; GFX10-DL-NEXT:    s_mov_b32 s15, 0x31c16000
902; GFX10-DL-NEXT:    s_add_u32 s12, s12, s3
903; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
904; GFX10-DL-NEXT:    s_addc_u32 s13, s13, 0
905; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
906; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
907; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
908; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
909; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
910; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
911; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 12
912; GFX10-DL-NEXT:    s_lshr_b32 s3, s1, 12
913; GFX10-DL-NEXT:    s_bfe_i32 s6, s0, 0x40000
914; GFX10-DL-NEXT:    s_bfe_i32 s7, s1, 0x40000
915; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v2, 12, s2
916; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s3
917; GFX10-DL-NEXT:    s_bfe_i32 s8, s0, 0x40004
918; GFX10-DL-NEXT:    s_bfe_i32 s9, s0, 0x40008
919; GFX10-DL-NEXT:    s_bfe_i32 s10, s1, 0x40008
920; GFX10-DL-NEXT:    s_bfe_i32 s2, s1, 0x40004
921; GFX10-DL-NEXT:    v_mul_i32_i24_e64 v4, s9, s10
922; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v2, 12, v2
923; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v3, 12, v3
924; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x40010
925; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
926; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s6, s7, v1
927; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s8, s2, v1
928; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
929; GFX10-DL-NEXT:    v_and_b32_e32 v2, s2, v2
930; GFX10-DL-NEXT:    v_and_b32_e32 v3, s2, v3
931; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
932; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40010
933; GFX10-DL-NEXT:    v_mad_u32_u24 v1, v2, v3, v1
934; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
935; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40014
936; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x40014
937; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
938; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40018
939; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x40018
940; GFX10-DL-NEXT:    s_ashr_i32 s0, s0, 28
941; GFX10-DL-NEXT:    s_ashr_i32 s1, s1, 28
942; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
943; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s0, s1, v1
944; GFX10-DL-NEXT:    global_store_byte v0, v1, s[4:5]
945; GFX10-DL-NEXT:    s_endpgm
946                                       <8 x i4> addrspace(1)* %src2,
947                                       i8 addrspace(1)* nocapture %dst) {
948entry:
949  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
950  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
951
952  %v1e0 = extractelement <8 x i4> %vec1, i64 0
953  %cv1e0 = sext i4 %v1e0 to i8
954  %v2e0 = extractelement <8 x i4> %vec2, i64 0
955  %cv2e0 = sext i4 %v2e0 to i8
956  %mul0 = mul nuw nsw i8 %cv1e0, %cv2e0
957
958  %v1e1 = extractelement <8 x i4> %vec1, i64 1
959  %cv1e1 = sext i4 %v1e1 to i8
960  %v2e1 = extractelement <8 x i4> %vec2, i64 1
961  %cv2e1 = sext i4 %v2e1 to i8
962  %mul1 = mul nuw nsw i8 %cv1e1, %cv2e1
963
964  %v1e2 = extractelement <8 x i4> %vec1, i64 2
965  %cv1e2 = sext i4 %v1e2 to i8
966  %v2e2 = extractelement <8 x i4> %vec2, i64 2
967  %cv2e2 = sext i4 %v2e2 to i8
968  %mul2 = mul nuw nsw i8 %cv1e2, %cv2e2
969
970  %v1e3 = extractelement <8 x i4> %vec1, i64 3
971  %cv1e3 = sext i4 %v1e3 to i8
972  %v2e3 = extractelement <8 x i4> %vec2, i64 3
973  %cv2e3 = sext i4 %v2e3 to i8
974  %mul3 = mul nuw nsw i8 %cv1e3, %cv2e3
975
976  %v1e4 = extractelement <8 x i4> %vec1, i64 4
977  %cv1e4 = sext i4 %v1e4 to i8
978  %v2e4 = extractelement <8 x i4> %vec2, i64 4
979  %cv2e4 = sext i4 %v2e4 to i8
980  %mul4 = mul nuw nsw i8 %cv1e4, %cv2e4
981
982  %v1e5 = extractelement <8 x i4> %vec1, i64 5
983  %cv1e5 = sext i4 %v1e5 to i8
984  %v2e5 = extractelement <8 x i4> %vec2, i64 5
985  %cv2e5 = sext i4 %v2e5 to i8
986  %mul5 = mul nuw nsw i8 %cv1e5, %cv2e5
987
988  %v1e6 = extractelement <8 x i4> %vec1, i64 6
989  %cv1e6 = sext i4 %v1e6 to i8
990  %v2e6 = extractelement <8 x i4> %vec2, i64 6
991  %cv2e6 = sext i4 %v2e6 to i8
992  %mul6 = mul nuw nsw i8 %cv1e6, %cv2e6
993
994  %v1e7 = extractelement <8 x i4> %vec1, i64 7
995  %cv1e7 = sext i4 %v1e7 to i8
996  %v2e7 = extractelement <8 x i4> %vec2, i64 7
997  %cv2e7 = sext i4 %v2e7 to i8
998  %mul7 = mul nuw nsw i8 %cv1e7, %cv2e7
999
1000  %acc = load i8, i8 addrspace(1)* %dst, align 4
1001  %add1 = add i8 %mul0, %acc
1002  %add2 = add i8 %add1, %mul1
1003  %add3 = add i8 %add2, %mul2
1004  %add4 = add i8 %add3, %mul3
1005  %add5 = add i8 %add4, %mul4
1006  %add6 = add i8 %add5, %mul5
1007  %add7 = add i8 %add6, %mul6
1008  %add8 = add i8 %add7, %mul7
1009
1010  store i8 %add8, i8 addrspace(1)* %dst, align 4
1011  ret void
1012}
1013
1014; Make sure the pattern is not recognized if there are multiple uses of the
1015; intermediate multiplications.
1016define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
1017; GFX7-LABEL: idot8_multiuses_mul1:
1018; GFX7:       ; %bb.0: ; %entry
1019; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1020; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1021; GFX7-NEXT:    s_mov_b32 s24, SCRATCH_RSRC_DWORD0
1022; GFX7-NEXT:    s_mov_b32 s25, SCRATCH_RSRC_DWORD1
1023; GFX7-NEXT:    s_mov_b32 s26, -1
1024; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1025; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1026; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1027; GFX7-NEXT:    s_load_dword s20, s[0:1], 0x0
1028; GFX7-NEXT:    s_mov_b32 s27, 0xe8f000
1029; GFX7-NEXT:    s_add_u32 s24, s24, s3
1030; GFX7-NEXT:    s_addc_u32 s25, s25, 0
1031; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1032; GFX7-NEXT:    s_bfe_i32 s7, s5, 0x40000
1033; GFX7-NEXT:    s_bfe_i32 s6, s4, 0x40000
1034; GFX7-NEXT:    v_mov_b32_e32 v0, s7
1035; GFX7-NEXT:    v_mov_b32_e32 v1, s20
1036; GFX7-NEXT:    v_mad_i32_i24 v1, s6, v0, v1
1037; GFX7-NEXT:    s_bfe_i32 s9, s5, 0x40004
1038; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x40004
1039; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x40008
1040; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v0, v1
1041; GFX7-NEXT:    v_mov_b32_e32 v2, s9
1042; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v2, v0
1043; GFX7-NEXT:    s_bfe_i32 s10, s4, 0x40008
1044; GFX7-NEXT:    v_mov_b32_e32 v2, s11
1045; GFX7-NEXT:    s_bfe_i32 s13, s5, 0x4000c
1046; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v2, v0
1047; GFX7-NEXT:    s_bfe_i32 s12, s4, 0x4000c
1048; GFX7-NEXT:    v_mov_b32_e32 v2, s13
1049; GFX7-NEXT:    s_bfe_i32 s15, s5, 0x40010
1050; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v2, v0
1051; GFX7-NEXT:    s_bfe_i32 s14, s4, 0x40010
1052; GFX7-NEXT:    v_mov_b32_e32 v2, s15
1053; GFX7-NEXT:    s_bfe_i32 s17, s5, 0x40014
1054; GFX7-NEXT:    s_bfe_i32 s19, s5, 0x40018
1055; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v2, v0
1056; GFX7-NEXT:    s_bfe_i32 s16, s4, 0x40014
1057; GFX7-NEXT:    v_mov_b32_e32 v2, s17
1058; GFX7-NEXT:    s_bfe_i32 s18, s4, 0x40018
1059; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v2, v0
1060; GFX7-NEXT:    v_mov_b32_e32 v2, s19
1061; GFX7-NEXT:    s_ashr_i32 s5, s5, 28
1062; GFX7-NEXT:    v_mad_i32_i24 v0, s18, v2, v0
1063; GFX7-NEXT:    s_ashr_i32 s4, s4, 28
1064; GFX7-NEXT:    v_mov_b32_e32 v2, s5
1065; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v2, v0
1066; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1067; GFX7-NEXT:    s_mov_b32 s2, -1
1068; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
1069; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1070; GFX7-NEXT:    s_endpgm
1071;
1072; GFX8-LABEL: idot8_multiuses_mul1:
1073; GFX8:       ; %bb.0: ; %entry
1074; GFX8-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
1075; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1076; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1077; GFX8-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
1078; GFX8-NEXT:    s_mov_b32 s22, -1
1079; GFX8-NEXT:    s_mov_b32 s23, 0xe80000
1080; GFX8-NEXT:    s_add_u32 s20, s20, s3
1081; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1082; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
1083; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
1084; GFX8-NEXT:    s_load_dword s18, s[0:1], 0x0
1085; GFX8-NEXT:    s_addc_u32 s21, s21, 0
1086; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1087; GFX8-NEXT:    s_bfe_i32 s4, s2, 0x40000
1088; GFX8-NEXT:    s_bfe_i32 s5, s3, 0x40000
1089; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1090; GFX8-NEXT:    v_mov_b32_e32 v1, s18
1091; GFX8-NEXT:    v_mad_i32_i24 v1, s4, v0, v1
1092; GFX8-NEXT:    s_bfe_i32 s7, s3, 0x40004
1093; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x40004
1094; GFX8-NEXT:    s_bfe_i32 s9, s3, 0x40008
1095; GFX8-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
1096; GFX8-NEXT:    v_mov_b32_e32 v2, s7
1097; GFX8-NEXT:    v_mad_i32_i24 v0, s6, v2, v0
1098; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x40008
1099; GFX8-NEXT:    v_mov_b32_e32 v2, s9
1100; GFX8-NEXT:    s_bfe_i32 s11, s3, 0x4000c
1101; GFX8-NEXT:    v_mad_i32_i24 v0, s8, v2, v0
1102; GFX8-NEXT:    s_bfe_i32 s10, s2, 0x4000c
1103; GFX8-NEXT:    v_mov_b32_e32 v2, s11
1104; GFX8-NEXT:    s_bfe_i32 s13, s3, 0x40010
1105; GFX8-NEXT:    v_mad_i32_i24 v0, s10, v2, v0
1106; GFX8-NEXT:    s_bfe_i32 s12, s2, 0x40010
1107; GFX8-NEXT:    v_mov_b32_e32 v2, s13
1108; GFX8-NEXT:    s_bfe_i32 s15, s3, 0x40014
1109; GFX8-NEXT:    s_bfe_i32 s17, s3, 0x40018
1110; GFX8-NEXT:    v_mad_i32_i24 v0, s12, v2, v0
1111; GFX8-NEXT:    s_bfe_i32 s14, s2, 0x40014
1112; GFX8-NEXT:    v_mov_b32_e32 v2, s15
1113; GFX8-NEXT:    s_bfe_i32 s16, s2, 0x40018
1114; GFX8-NEXT:    v_mad_i32_i24 v0, s14, v2, v0
1115; GFX8-NEXT:    v_mov_b32_e32 v2, s17
1116; GFX8-NEXT:    s_ashr_i32 s3, s3, 28
1117; GFX8-NEXT:    v_mad_i32_i24 v0, s16, v2, v0
1118; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
1119; GFX8-NEXT:    v_mov_b32_e32 v2, s3
1120; GFX8-NEXT:    v_mad_i32_i24 v0, s2, v2, v0
1121; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v1
1122; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1123; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1124; GFX8-NEXT:    flat_store_dword v[0:1], v2
1125; GFX8-NEXT:    s_endpgm
1126;
1127; GFX9-LABEL: idot8_multiuses_mul1:
1128; GFX9:       ; %bb.0: ; %entry
1129; GFX9-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
1130; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1131; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1132; GFX9-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
1133; GFX9-NEXT:    s_mov_b32 s22, -1
1134; GFX9-NEXT:    s_mov_b32 s23, 0xe00000
1135; GFX9-NEXT:    s_add_u32 s20, s20, s3
1136; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1137; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
1138; GFX9-NEXT:    s_load_dword s3, s[6:7], 0x0
1139; GFX9-NEXT:    s_load_dword s18, s[0:1], 0x0
1140; GFX9-NEXT:    s_addc_u32 s21, s21, 0
1141; GFX9-NEXT:    v_mov_b32_e32 v0, 0
1142; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1143; GFX9-NEXT:    s_bfe_i32 s4, s2, 0x40000
1144; GFX9-NEXT:    s_bfe_i32 s5, s3, 0x40000
1145; GFX9-NEXT:    v_mov_b32_e32 v1, s5
1146; GFX9-NEXT:    v_mov_b32_e32 v2, s18
1147; GFX9-NEXT:    v_mad_i32_i24 v2, s4, v1, v2
1148; GFX9-NEXT:    s_bfe_i32 s7, s3, 0x40004
1149; GFX9-NEXT:    s_bfe_i32 s6, s2, 0x40004
1150; GFX9-NEXT:    s_bfe_i32 s9, s3, 0x40008
1151; GFX9-NEXT:    v_mad_i32_i24 v1, s4, v1, v2
1152; GFX9-NEXT:    v_mov_b32_e32 v3, s7
1153; GFX9-NEXT:    v_mad_i32_i24 v1, s6, v3, v1
1154; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x40008
1155; GFX9-NEXT:    v_mov_b32_e32 v3, s9
1156; GFX9-NEXT:    s_bfe_i32 s11, s3, 0x4000c
1157; GFX9-NEXT:    v_mad_i32_i24 v1, s8, v3, v1
1158; GFX9-NEXT:    s_bfe_i32 s10, s2, 0x4000c
1159; GFX9-NEXT:    v_mov_b32_e32 v3, s11
1160; GFX9-NEXT:    s_bfe_i32 s13, s3, 0x40010
1161; GFX9-NEXT:    v_mad_i32_i24 v1, s10, v3, v1
1162; GFX9-NEXT:    s_bfe_i32 s12, s2, 0x40010
1163; GFX9-NEXT:    v_mov_b32_e32 v3, s13
1164; GFX9-NEXT:    s_bfe_i32 s15, s3, 0x40014
1165; GFX9-NEXT:    s_bfe_i32 s17, s3, 0x40018
1166; GFX9-NEXT:    v_mad_i32_i24 v1, s12, v3, v1
1167; GFX9-NEXT:    s_bfe_i32 s14, s2, 0x40014
1168; GFX9-NEXT:    v_mov_b32_e32 v3, s15
1169; GFX9-NEXT:    s_bfe_i32 s16, s2, 0x40018
1170; GFX9-NEXT:    v_mad_i32_i24 v1, s14, v3, v1
1171; GFX9-NEXT:    v_mov_b32_e32 v3, s17
1172; GFX9-NEXT:    s_ashr_i32 s3, s3, 28
1173; GFX9-NEXT:    v_mad_i32_i24 v1, s16, v3, v1
1174; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
1175; GFX9-NEXT:    v_mov_b32_e32 v3, s3
1176; GFX9-NEXT:    v_mad_i32_i24 v1, s2, v3, v1
1177; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
1178; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
1179; GFX9-NEXT:    s_endpgm
1180;
1181; GFX9-DL-LABEL: idot8_multiuses_mul1:
1182; GFX9-DL:       ; %bb.0: ; %entry
1183; GFX9-DL-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
1184; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1185; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1186; GFX9-DL-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
1187; GFX9-DL-NEXT:    s_mov_b32 s22, -1
1188; GFX9-DL-NEXT:    s_mov_b32 s23, 0xe00000
1189; GFX9-DL-NEXT:    s_add_u32 s20, s20, s3
1190; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1191; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1192; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1193; GFX9-DL-NEXT:    s_load_dword s18, s[0:1], 0x0
1194; GFX9-DL-NEXT:    s_addc_u32 s21, s21, 0
1195; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
1196; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1197; GFX9-DL-NEXT:    s_bfe_i32 s4, s2, 0x40000
1198; GFX9-DL-NEXT:    s_bfe_i32 s5, s3, 0x40000
1199; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s5
1200; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s18
1201; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s4, v1, v2
1202; GFX9-DL-NEXT:    s_bfe_i32 s7, s3, 0x40004
1203; GFX9-DL-NEXT:    s_bfe_i32 s6, s2, 0x40004
1204; GFX9-DL-NEXT:    s_bfe_i32 s9, s3, 0x40008
1205; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s4, v1, v2
1206; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s7
1207; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s6, v3, v1
1208; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x40008
1209; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s9
1210; GFX9-DL-NEXT:    s_bfe_i32 s11, s3, 0x4000c
1211; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s8, v3, v1
1212; GFX9-DL-NEXT:    s_bfe_i32 s10, s2, 0x4000c
1213; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s11
1214; GFX9-DL-NEXT:    s_bfe_i32 s13, s3, 0x40010
1215; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s10, v3, v1
1216; GFX9-DL-NEXT:    s_bfe_i32 s12, s2, 0x40010
1217; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s13
1218; GFX9-DL-NEXT:    s_bfe_i32 s15, s3, 0x40014
1219; GFX9-DL-NEXT:    s_bfe_i32 s17, s3, 0x40018
1220; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s12, v3, v1
1221; GFX9-DL-NEXT:    s_bfe_i32 s14, s2, 0x40014
1222; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s15
1223; GFX9-DL-NEXT:    s_bfe_i32 s16, s2, 0x40018
1224; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s14, v3, v1
1225; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s17
1226; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 28
1227; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s16, v3, v1
1228; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 28
1229; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
1230; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s2, v3, v1
1231; GFX9-DL-NEXT:    v_add_u32_e32 v1, v2, v1
1232; GFX9-DL-NEXT:    global_store_dword v0, v1, s[0:1]
1233; GFX9-DL-NEXT:    s_endpgm
1234;
1235; GFX10-DL-LABEL: idot8_multiuses_mul1:
1236; GFX10-DL:       ; %bb.0: ; %entry
1237; GFX10-DL-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
1238; GFX10-DL-NEXT:    s_mov_b32 s9, SCRATCH_RSRC_DWORD1
1239; GFX10-DL-NEXT:    s_mov_b32 s10, -1
1240; GFX10-DL-NEXT:    s_mov_b32 s11, 0x31c16000
1241; GFX10-DL-NEXT:    s_add_u32 s8, s8, s3
1242; GFX10-DL-NEXT:    s_clause 0x1
1243; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1244; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1245; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
1246; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0
1247; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1248; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1249; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
1250; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
1251; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
1252; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1253; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1254; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40000
1255; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x40000
1256; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s2, s3, v0
1257; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v0
1258; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40004
1259; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x40004
1260; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
1261; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40008
1262; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x40008
1263; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
1264; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x4000c
1265; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x4000c
1266; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
1267; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40010
1268; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x40010
1269; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
1270; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40014
1271; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x40014
1272; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
1273; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x40018
1274; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x40018
1275; GFX10-DL-NEXT:    s_ashr_i32 s0, s0, 28
1276; GFX10-DL-NEXT:    s_ashr_i32 s1, s1, 28
1277; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
1278; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s0, s1, v1
1279; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v0, v1
1280; GFX10-DL-NEXT:    global_store_dword v2, v0, s[4:5]
1281; GFX10-DL-NEXT:    s_endpgm
1282                                                <8 x i4> addrspace(1)* %src2,
1283                                                i32 addrspace(1)* nocapture %dst) {
1284entry:
1285  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1286  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1287
1288  %v1e0 = extractelement <8 x i4> %vec1, i64 0
1289  %cv1e0 = sext i4 %v1e0 to i32
1290  %v2e0 = extractelement <8 x i4> %vec2, i64 0
1291  %cv2e0 = sext i4 %v2e0 to i32
1292  %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
1293
1294  %v1e1 = extractelement <8 x i4> %vec1, i64 1
1295  %cv1e1 = sext i4 %v1e1 to i32
1296  %v2e1 = extractelement <8 x i4> %vec2, i64 1
1297  %cv2e1 = sext i4 %v2e1 to i32
1298  %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
1299
1300  %v1e2 = extractelement <8 x i4> %vec1, i64 2
1301  %cv1e2 = sext i4 %v1e2 to i32
1302  %v2e2 = extractelement <8 x i4> %vec2, i64 2
1303  %cv2e2 = sext i4 %v2e2 to i32
1304  %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
1305
1306  %v1e3 = extractelement <8 x i4> %vec1, i64 3
1307  %cv1e3 = sext i4 %v1e3 to i32
1308  %v2e3 = extractelement <8 x i4> %vec2, i64 3
1309  %cv2e3 = sext i4 %v2e3 to i32
1310  %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
1311
1312  %v1e4 = extractelement <8 x i4> %vec1, i64 4
1313  %cv1e4 = sext i4 %v1e4 to i32
1314  %v2e4 = extractelement <8 x i4> %vec2, i64 4
1315  %cv2e4 = sext i4 %v2e4 to i32
1316  %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
1317
1318  %v1e5 = extractelement <8 x i4> %vec1, i64 5
1319  %cv1e5 = sext i4 %v1e5 to i32
1320  %v2e5 = extractelement <8 x i4> %vec2, i64 5
1321  %cv2e5 = sext i4 %v2e5 to i32
1322  %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
1323
1324  %v1e6 = extractelement <8 x i4> %vec1, i64 6
1325  %cv1e6 = sext i4 %v1e6 to i32
1326  %v2e6 = extractelement <8 x i4> %vec2, i64 6
1327  %cv2e6 = sext i4 %v2e6 to i32
1328  %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
1329
1330  %v1e7 = extractelement <8 x i4> %vec1, i64 7
1331  %cv1e7 = sext i4 %v1e7 to i32
1332  %v2e7 = extractelement <8 x i4> %vec2, i64 7
1333  %cv2e7 = sext i4 %v2e7 to i32
1334  %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
1335
1336  %acc = load i32, i32 addrspace(1)* %dst, align 4
1337  %add =  add i32  %mul0, %acc
1338  %add1 = add i32 %mul0, %add
1339  %add2 = add i32 %add1, %mul1
1340  %add3 = add i32 %add2, %mul2
1341  %add4 = add i32 %add3, %mul3
1342  %add5 = add i32 %add4, %mul4
1343  %add6 = add i32 %add5, %mul5
1344  %add7 = add i32 %add6, %mul6
1345  %add8 = add i32 %add7, %mul7
1346
1347  %res = add i32 %add, %add8
1348  store i32 %res, i32 addrspace(1)* %dst, align 4
1349  ret void
1350}
1351
1352; TODO: Support this pattern.
1353define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
1354; GFX7-LABEL: idot8_acc32_vecMul:
1355; GFX7:       ; %bb.0: ; %entry
1356; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1357; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1358; GFX7-NEXT:    s_mov_b32 s24, SCRATCH_RSRC_DWORD0
1359; GFX7-NEXT:    s_mov_b32 s25, SCRATCH_RSRC_DWORD1
1360; GFX7-NEXT:    s_mov_b32 s26, -1
1361; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1362; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1363; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1364; GFX7-NEXT:    s_load_dword s20, s[0:1], 0x0
1365; GFX7-NEXT:    s_mov_b32 s27, 0xe8f000
1366; GFX7-NEXT:    s_add_u32 s24, s24, s3
1367; GFX7-NEXT:    s_addc_u32 s25, s25, 0
1368; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1369; GFX7-NEXT:    s_ashr_i32 s13, s5, 28
1370; GFX7-NEXT:    s_bfe_i32 s14, s5, 0x40018
1371; GFX7-NEXT:    s_bfe_i32 s15, s5, 0x40014
1372; GFX7-NEXT:    s_bfe_i32 s16, s5, 0x40010
1373; GFX7-NEXT:    s_bfe_i32 s17, s5, 0x4000c
1374; GFX7-NEXT:    s_bfe_i32 s18, s5, 0x40008
1375; GFX7-NEXT:    s_bfe_i32 s19, s5, 0x40004
1376; GFX7-NEXT:    s_bfe_i32 s5, s5, 0x40000
1377; GFX7-NEXT:    s_ashr_i32 s6, s4, 28
1378; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x40018
1379; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x40014
1380; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x40010
1381; GFX7-NEXT:    s_bfe_i32 s10, s4, 0x4000c
1382; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x40008
1383; GFX7-NEXT:    s_bfe_i32 s12, s4, 0x40004
1384; GFX7-NEXT:    s_bfe_i32 s4, s4, 0x40000
1385; GFX7-NEXT:    v_mov_b32_e32 v0, s5
1386; GFX7-NEXT:    v_mov_b32_e32 v1, s20
1387; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
1388; GFX7-NEXT:    v_mov_b32_e32 v1, s19
1389; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
1390; GFX7-NEXT:    v_mov_b32_e32 v1, s18
1391; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
1392; GFX7-NEXT:    v_mov_b32_e32 v1, s17
1393; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
1394; GFX7-NEXT:    v_mov_b32_e32 v1, s16
1395; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
1396; GFX7-NEXT:    v_mov_b32_e32 v1, s15
1397; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
1398; GFX7-NEXT:    v_mov_b32_e32 v1, s14
1399; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v1, v0
1400; GFX7-NEXT:    v_mov_b32_e32 v1, s13
1401; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1402; GFX7-NEXT:    s_mov_b32 s2, -1
1403; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
1404; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1405; GFX7-NEXT:    s_endpgm
1406;
1407; GFX8-LABEL: idot8_acc32_vecMul:
1408; GFX8:       ; %bb.0: ; %entry
1409; GFX8-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
1410; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1411; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1412; GFX8-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
1413; GFX8-NEXT:    s_mov_b32 s22, -1
1414; GFX8-NEXT:    s_mov_b32 s23, 0xe80000
1415; GFX8-NEXT:    s_add_u32 s20, s20, s3
1416; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1417; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
1418; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
1419; GFX8-NEXT:    s_load_dword s18, s[0:1], 0x0
1420; GFX8-NEXT:    s_addc_u32 s21, s21, 0
1421; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1422; GFX8-NEXT:    s_ashr_i32 s4, s2, 28
1423; GFX8-NEXT:    s_ashr_i32 s11, s3, 28
1424; GFX8-NEXT:    s_bfe_i32 s12, s3, 0x40018
1425; GFX8-NEXT:    s_bfe_i32 s13, s3, 0x40014
1426; GFX8-NEXT:    s_bfe_i32 s14, s3, 0x40010
1427; GFX8-NEXT:    s_bfe_i32 s15, s3, 0x4000c
1428; GFX8-NEXT:    s_bfe_i32 s16, s3, 0x40008
1429; GFX8-NEXT:    s_bfe_i32 s17, s3, 0x40004
1430; GFX8-NEXT:    s_bfe_i32 s3, s3, 0x40000
1431; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x40018
1432; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x40014
1433; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x40010
1434; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x4000c
1435; GFX8-NEXT:    s_bfe_i32 s9, s2, 0x40008
1436; GFX8-NEXT:    s_bfe_i32 s10, s2, 0x40004
1437; GFX8-NEXT:    s_bfe_i32 s2, s2, 0x40000
1438; GFX8-NEXT:    v_mov_b32_e32 v0, s3
1439; GFX8-NEXT:    v_mov_b32_e32 v1, s18
1440; GFX8-NEXT:    v_mad_i32_i24 v0, s2, v0, v1
1441; GFX8-NEXT:    v_mov_b32_e32 v1, s17
1442; GFX8-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
1443; GFX8-NEXT:    v_mov_b32_e32 v1, s16
1444; GFX8-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
1445; GFX8-NEXT:    v_mov_b32_e32 v1, s15
1446; GFX8-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
1447; GFX8-NEXT:    v_mov_b32_e32 v1, s14
1448; GFX8-NEXT:    v_mad_i32_i24 v0, s7, v1, v0
1449; GFX8-NEXT:    v_mov_b32_e32 v1, s13
1450; GFX8-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
1451; GFX8-NEXT:    v_mov_b32_e32 v1, s12
1452; GFX8-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
1453; GFX8-NEXT:    v_mov_b32_e32 v1, s11
1454; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v1, v0
1455; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1456; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1457; GFX8-NEXT:    flat_store_dword v[0:1], v2
1458; GFX8-NEXT:    s_endpgm
1459;
1460; GFX9-LABEL: idot8_acc32_vecMul:
1461; GFX9:       ; %bb.0: ; %entry
1462; GFX9-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
1463; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1464; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1465; GFX9-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
1466; GFX9-NEXT:    s_mov_b32 s22, -1
1467; GFX9-NEXT:    s_mov_b32 s23, 0xe00000
1468; GFX9-NEXT:    s_add_u32 s20, s20, s3
1469; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1470; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
1471; GFX9-NEXT:    s_load_dword s3, s[6:7], 0x0
1472; GFX9-NEXT:    s_load_dword s18, s[0:1], 0x0
1473; GFX9-NEXT:    s_addc_u32 s21, s21, 0
1474; GFX9-NEXT:    v_mov_b32_e32 v0, 0
1475; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1476; GFX9-NEXT:    s_ashr_i32 s4, s2, 28
1477; GFX9-NEXT:    s_ashr_i32 s11, s3, 28
1478; GFX9-NEXT:    s_bfe_i32 s12, s3, 0x40018
1479; GFX9-NEXT:    s_bfe_i32 s13, s3, 0x40014
1480; GFX9-NEXT:    s_bfe_i32 s14, s3, 0x40010
1481; GFX9-NEXT:    s_bfe_i32 s15, s3, 0x4000c
1482; GFX9-NEXT:    s_bfe_i32 s16, s3, 0x40008
1483; GFX9-NEXT:    s_bfe_i32 s17, s3, 0x40004
1484; GFX9-NEXT:    s_bfe_i32 s3, s3, 0x40000
1485; GFX9-NEXT:    s_bfe_i32 s5, s2, 0x40018
1486; GFX9-NEXT:    s_bfe_i32 s6, s2, 0x40014
1487; GFX9-NEXT:    s_bfe_i32 s7, s2, 0x40010
1488; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x4000c
1489; GFX9-NEXT:    s_bfe_i32 s9, s2, 0x40008
1490; GFX9-NEXT:    s_bfe_i32 s10, s2, 0x40004
1491; GFX9-NEXT:    s_bfe_i32 s2, s2, 0x40000
1492; GFX9-NEXT:    v_mov_b32_e32 v1, s3
1493; GFX9-NEXT:    v_mov_b32_e32 v2, s18
1494; GFX9-NEXT:    v_mad_i32_i24 v1, s2, v1, v2
1495; GFX9-NEXT:    v_mov_b32_e32 v2, s17
1496; GFX9-NEXT:    v_mad_i32_i24 v1, s10, v2, v1
1497; GFX9-NEXT:    v_mov_b32_e32 v2, s16
1498; GFX9-NEXT:    v_mad_i32_i24 v1, s9, v2, v1
1499; GFX9-NEXT:    v_mov_b32_e32 v2, s15
1500; GFX9-NEXT:    v_mad_i32_i24 v1, s8, v2, v1
1501; GFX9-NEXT:    v_mov_b32_e32 v2, s14
1502; GFX9-NEXT:    v_mad_i32_i24 v1, s7, v2, v1
1503; GFX9-NEXT:    v_mov_b32_e32 v2, s13
1504; GFX9-NEXT:    v_mad_i32_i24 v1, s6, v2, v1
1505; GFX9-NEXT:    v_mov_b32_e32 v2, s12
1506; GFX9-NEXT:    v_mad_i32_i24 v1, s5, v2, v1
1507; GFX9-NEXT:    v_mov_b32_e32 v2, s11
1508; GFX9-NEXT:    v_mad_i32_i24 v1, s4, v2, v1
1509; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
1510; GFX9-NEXT:    s_endpgm
1511;
1512; GFX9-DL-LABEL: idot8_acc32_vecMul:
1513; GFX9-DL:       ; %bb.0: ; %entry
1514; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1515; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1516; GFX9-DL-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
1517; GFX9-DL-NEXT:    s_mov_b32 s9, SCRATCH_RSRC_DWORD1
1518; GFX9-DL-NEXT:    s_mov_b32 s10, -1
1519; GFX9-DL-NEXT:    s_mov_b32 s11, 0xe00000
1520; GFX9-DL-NEXT:    s_add_u32 s8, s8, s3
1521; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1522; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
1523; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
1524; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
1525; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
1526; GFX9-DL-NEXT:    s_addc_u32 s9, s9, 0
1527; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1528; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
1529; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
1530; GFX9-DL-NEXT:    v_dot8_i32_i4 v1, s4, v1, v2
1531; GFX9-DL-NEXT:    global_store_dword v0, v1, s[0:1]
1532; GFX9-DL-NEXT:    s_endpgm
1533;
1534; GFX10-DL-LABEL: idot8_acc32_vecMul:
1535; GFX10-DL:       ; %bb.0: ; %entry
1536; GFX10-DL-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
1537; GFX10-DL-NEXT:    s_mov_b32 s9, SCRATCH_RSRC_DWORD1
1538; GFX10-DL-NEXT:    s_mov_b32 s10, -1
1539; GFX10-DL-NEXT:    s_mov_b32 s11, 0x31c16000
1540; GFX10-DL-NEXT:    s_add_u32 s8, s8, s3
1541; GFX10-DL-NEXT:    s_clause 0x1
1542; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1543; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1544; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
1545; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
1546; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1547; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1548; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
1549; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
1550; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
1551; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1552; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1553; GFX10-DL-NEXT:    v_dot8_i32_i4 v0, s0, s1, v0
1554; GFX10-DL-NEXT:    global_store_dword v1, v0, s[4:5]
1555; GFX10-DL-NEXT:    s_endpgm
1556                                              <8 x i4> addrspace(1)* %src2,
1557                                              i32 addrspace(1)* nocapture %dst) {
1558entry:
1559  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1560  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1561
1562  %cvec1 = sext <8 x i4> %vec1 to <8 x i32>
1563  %cvec2 = sext <8 x i4> %vec2 to <8 x i32>
1564
1565  %mul = mul <8 x i32> %cvec1, %cvec2
1566  %mul0 = extractelement <8 x i32> %mul, i64 0
1567  %mul1 = extractelement <8 x i32> %mul, i64 1
1568  %mul2 = extractelement <8 x i32> %mul, i64 2
1569  %mul3 = extractelement <8 x i32> %mul, i64 3
1570  %mul4 = extractelement <8 x i32> %mul, i64 4
1571  %mul5 = extractelement <8 x i32> %mul, i64 5
1572  %mul6 = extractelement <8 x i32> %mul, i64 6
1573  %mul7 = extractelement <8 x i32> %mul, i64 7
1574
1575  %acc = load i32, i32 addrspace(1)* %dst, align 4
1576  %add1 = add i32 %mul0, %acc
1577  %add2 = add i32 %add1, %mul1
1578  %add3 = add i32 %add2, %mul2
1579  %add4 = add i32 %add3, %mul3
1580  %add5 = add i32 %add4, %mul4
1581  %add6 = add i32 %add5, %mul5
1582  %add7 = add i32 %add6, %mul6
1583  %add8 = add i32 %add7, %mul7
1584
1585  store i32 %add8, i32 addrspace(1)* %dst, align 4
1586  ret void
1587}
1588
1589; TODO: Support this pattern.
1590define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
1591; GFX7-LABEL: idot8_acc16_vecMul:
1592; GFX7:       ; %bb.0: ; %entry
1593; GFX7-NEXT:    s_mov_b32 s24, SCRATCH_RSRC_DWORD0
1594; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1595; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1596; GFX7-NEXT:    s_mov_b32 s25, SCRATCH_RSRC_DWORD1
1597; GFX7-NEXT:    s_mov_b32 s26, -1
1598; GFX7-NEXT:    s_mov_b32 s27, 0xe8f000
1599; GFX7-NEXT:    s_add_u32 s24, s24, s3
1600; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1601; GFX7-NEXT:    s_mov_b32 s2, -1
1602; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1603; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
1604; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1605; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1606; GFX7-NEXT:    s_addc_u32 s25, s25, 0
1607; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1608; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1609; GFX7-NEXT:    s_ashr_i32 s6, s4, 28
1610; GFX7-NEXT:    s_bfe_i32 s15, s5, 0x40018
1611; GFX7-NEXT:    s_bfe_i32 s16, s5, 0x40014
1612; GFX7-NEXT:    s_bfe_i32 s17, s5, 0x40010
1613; GFX7-NEXT:    s_bfe_i32 s18, s5, 0x40008
1614; GFX7-NEXT:    s_bfe_i32 s19, s5, 0x4000c
1615; GFX7-NEXT:    s_bfe_i32 s20, s5, 0x40000
1616; GFX7-NEXT:    s_ashr_i32 s14, s5, 28
1617; GFX7-NEXT:    s_bfe_i32 s5, s5, 0x40004
1618; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x40018
1619; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x40014
1620; GFX7-NEXT:    s_bfe_i32 s10, s4, 0x40010
1621; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x40008
1622; GFX7-NEXT:    v_mov_b32_e32 v4, s18
1623; GFX7-NEXT:    s_bfe_i32 s12, s4, 0x4000c
1624; GFX7-NEXT:    v_mov_b32_e32 v3, s19
1625; GFX7-NEXT:    s_bfe_i32 s13, s4, 0x40000
1626; GFX7-NEXT:    v_mov_b32_e32 v2, s20
1627; GFX7-NEXT:    s_bfe_i32 s4, s4, 0x40004
1628; GFX7-NEXT:    v_mov_b32_e32 v1, s5
1629; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s4, v1
1630; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s13, v2
1631; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s12, v3
1632; GFX7-NEXT:    v_mul_i32_i24_e32 v4, s11, v4
1633; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
1634; GFX7-NEXT:    v_and_b32_e32 v2, s8, v2
1635; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
1636; GFX7-NEXT:    v_and_b32_e32 v4, s8, v4
1637; GFX7-NEXT:    v_or_b32_e32 v3, v4, v3
1638; GFX7-NEXT:    v_or_b32_e32 v2, v2, v1
1639; GFX7-NEXT:    v_alignbit_b32 v1, v3, v1, 16
1640; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
1641; GFX7-NEXT:    v_mov_b32_e32 v5, s17
1642; GFX7-NEXT:    v_mov_b32_e32 v6, s16
1643; GFX7-NEXT:    v_mov_b32_e32 v7, s15
1644; GFX7-NEXT:    s_waitcnt vmcnt(0)
1645; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
1646; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
1647; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
1648; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
1649; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v5, v0
1650; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v6, v0
1651; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v7, v0
1652; GFX7-NEXT:    v_mov_b32_e32 v1, s14
1653; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
1654; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
1655; GFX7-NEXT:    s_endpgm
1656;
1657; GFX8-LABEL: idot8_acc16_vecMul:
1658; GFX8:       ; %bb.0: ; %entry
1659; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1660; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1661; GFX8-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
1662; GFX8-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
1663; GFX8-NEXT:    s_mov_b32 s18, -1
1664; GFX8-NEXT:    s_mov_b32 s19, 0xe80000
1665; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1666; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1667; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1668; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
1669; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
1670; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
1671; GFX8-NEXT:    s_add_u32 s16, s16, s3
1672; GFX8-NEXT:    s_addc_u32 s17, s17, 0
1673; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1674; GFX8-NEXT:    s_bfe_i32 s8, s0, 0x40000
1675; GFX8-NEXT:    s_bfe_i32 s15, s1, 0x40000
1676; GFX8-NEXT:    s_bfe_i32 s10, s1, 0x40018
1677; GFX8-NEXT:    s_bfe_i32 s11, s1, 0x40014
1678; GFX8-NEXT:    s_bfe_i32 s12, s1, 0x40010
1679; GFX8-NEXT:    s_bfe_i32 s13, s1, 0x4000c
1680; GFX8-NEXT:    s_bfe_i32 s14, s1, 0x40004
1681; GFX8-NEXT:    s_ashr_i32 s9, s1, 28
1682; GFX8-NEXT:    s_bfe_i32 s1, s1, 0x40008
1683; GFX8-NEXT:    v_mov_b32_e32 v4, s15
1684; GFX8-NEXT:    s_ashr_i32 s2, s0, 28
1685; GFX8-NEXT:    s_bfe_i32 s3, s0, 0x40018
1686; GFX8-NEXT:    s_bfe_i32 s4, s0, 0x40014
1687; GFX8-NEXT:    s_bfe_i32 s5, s0, 0x40010
1688; GFX8-NEXT:    s_bfe_i32 s6, s0, 0x4000c
1689; GFX8-NEXT:    s_bfe_i32 s7, s0, 0x40004
1690; GFX8-NEXT:    s_bfe_i32 s0, s0, 0x40008
1691; GFX8-NEXT:    v_mov_b32_e32 v3, s1
1692; GFX8-NEXT:    v_mov_b32_e32 v5, s14
1693; GFX8-NEXT:    v_mul_i32_i24_e32 v3, s0, v3
1694; GFX8-NEXT:    v_mov_b32_e32 v6, s13
1695; GFX8-NEXT:    v_mov_b32_e32 v7, s12
1696; GFX8-NEXT:    v_mov_b32_e32 v8, s11
1697; GFX8-NEXT:    v_mov_b32_e32 v9, s10
1698; GFX8-NEXT:    s_waitcnt vmcnt(0)
1699; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v4, v2
1700; GFX8-NEXT:    v_mad_i32_i24 v2, s7, v5, v2
1701; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
1702; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v6, v2
1703; GFX8-NEXT:    v_mad_i32_i24 v2, s5, v7, v2
1704; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v8, v2
1705; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v9, v2
1706; GFX8-NEXT:    v_mov_b32_e32 v3, s9
1707; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
1708; GFX8-NEXT:    flat_store_short v[0:1], v2
1709; GFX8-NEXT:    s_endpgm
1710;
1711; GFX9-LABEL: idot8_acc16_vecMul:
1712; GFX9:       ; %bb.0: ; %entry
1713; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1714; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1715; GFX9-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
1716; GFX9-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
1717; GFX9-NEXT:    s_mov_b32 s22, -1
1718; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1719; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
1720; GFX9-NEXT:    s_mov_b32 s23, 0xe00000
1721; GFX9-NEXT:    s_add_u32 s20, s20, s3
1722; GFX9-NEXT:    s_addc_u32 s21, s21, 0
1723; GFX9-NEXT:    s_load_dword s6, s[6:7], 0x0
1724; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1725; GFX9-NEXT:    s_bfe_u32 s3, s2, 0x40018
1726; GFX9-NEXT:    s_lshr_b32 s4, s2, 28
1727; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40010
1728; GFX9-NEXT:    s_bfe_u32 s8, s2, 0x40014
1729; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40008
1730; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x4000c
1731; GFX9-NEXT:    s_and_b32 s11, s2, 15
1732; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x40004
1733; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s11, s2
1734; GFX9-NEXT:    v_pk_lshlrev_b16 v1, 12, s2 op_sel_hi:[0,1]
1735; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s9, s10
1736; GFX9-NEXT:    v_pk_lshlrev_b16 v2, 12, s2 op_sel_hi:[0,1]
1737; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s5, s8
1738; GFX9-NEXT:    v_pk_lshlrev_b16 v3, 12, s2 op_sel_hi:[0,1]
1739; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s3, s4
1740; GFX9-NEXT:    s_bfe_u32 s7, s6, 0x40018
1741; GFX9-NEXT:    s_lshr_b32 s12, s6, 28
1742; GFX9-NEXT:    s_bfe_u32 s13, s6, 0x40010
1743; GFX9-NEXT:    s_bfe_u32 s14, s6, 0x40014
1744; GFX9-NEXT:    s_bfe_u32 s15, s6, 0x40008
1745; GFX9-NEXT:    s_bfe_u32 s16, s6, 0x4000c
1746; GFX9-NEXT:    s_and_b32 s17, s6, 15
1747; GFX9-NEXT:    s_bfe_u32 s6, s6, 0x40004
1748; GFX9-NEXT:    v_pk_lshlrev_b16 v4, 12, s2 op_sel_hi:[0,1]
1749; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s17, s6
1750; GFX9-NEXT:    v_pk_lshlrev_b16 v5, 12, s2 op_sel_hi:[0,1]
1751; GFX9-NEXT:    v_mov_b32_e32 v0, 0
1752; GFX9-NEXT:    v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
1753; GFX9-NEXT:    v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
1754; GFX9-NEXT:    v_pk_mul_lo_u16 v1, v1, v5
1755; GFX9-NEXT:    global_load_ushort v5, v0, s[0:1]
1756; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s15, s16
1757; GFX9-NEXT:    v_pk_lshlrev_b16 v6, 12, s2 op_sel_hi:[0,1]
1758; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s13, s14
1759; GFX9-NEXT:    v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
1760; GFX9-NEXT:    v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
1761; GFX9-NEXT:    v_pk_lshlrev_b16 v7, 12, s2 op_sel_hi:[0,1]
1762; GFX9-NEXT:    v_pk_mul_lo_u16 v2, v2, v6
1763; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s7, s12
1764; GFX9-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
1765; GFX9-NEXT:    v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
1766; GFX9-NEXT:    v_pk_lshlrev_b16 v8, 12, s2 op_sel_hi:[0,1]
1767; GFX9-NEXT:    v_pk_mul_lo_u16 v3, v3, v7
1768; GFX9-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
1769; GFX9-NEXT:    v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
1770; GFX9-NEXT:    v_pk_mul_lo_u16 v4, v4, v8
1771; GFX9-NEXT:    s_waitcnt vmcnt(0)
1772; GFX9-NEXT:    v_add_u32_e32 v5, v1, v5
1773; GFX9-NEXT:    v_add_u32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1774; GFX9-NEXT:    v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
1775; GFX9-NEXT:    v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1776; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
1777; GFX9-NEXT:    v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1778; GFX9-NEXT:    v_add_u32_e32 v1, v1, v4
1779; GFX9-NEXT:    v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1780; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
1781; GFX9-NEXT:    s_endpgm
1782;
1783; GFX9-DL-LABEL: idot8_acc16_vecMul:
1784; GFX9-DL:       ; %bb.0: ; %entry
1785; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1786; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1787; GFX9-DL-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
1788; GFX9-DL-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
1789; GFX9-DL-NEXT:    s_mov_b32 s22, -1
1790; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1791; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1792; GFX9-DL-NEXT:    s_mov_b32 s23, 0xe00000
1793; GFX9-DL-NEXT:    s_add_u32 s20, s20, s3
1794; GFX9-DL-NEXT:    s_addc_u32 s21, s21, 0
1795; GFX9-DL-NEXT:    s_load_dword s6, s[6:7], 0x0
1796; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1797; GFX9-DL-NEXT:    s_bfe_u32 s3, s2, 0x40018
1798; GFX9-DL-NEXT:    s_lshr_b32 s4, s2, 28
1799; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x40010
1800; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x40014
1801; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40008
1802; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x4000c
1803; GFX9-DL-NEXT:    s_and_b32 s11, s2, 15
1804; GFX9-DL-NEXT:    s_bfe_u32 s2, s2, 0x40004
1805; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s11, s2
1806; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v1, 12, s2 op_sel_hi:[0,1]
1807; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s9, s10
1808; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v2, 12, s2 op_sel_hi:[0,1]
1809; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s5, s8
1810; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v3, 12, s2 op_sel_hi:[0,1]
1811; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s3, s4
1812; GFX9-DL-NEXT:    s_bfe_u32 s7, s6, 0x40018
1813; GFX9-DL-NEXT:    s_lshr_b32 s12, s6, 28
1814; GFX9-DL-NEXT:    s_bfe_u32 s13, s6, 0x40010
1815; GFX9-DL-NEXT:    s_bfe_u32 s14, s6, 0x40014
1816; GFX9-DL-NEXT:    s_bfe_u32 s15, s6, 0x40008
1817; GFX9-DL-NEXT:    s_bfe_u32 s16, s6, 0x4000c
1818; GFX9-DL-NEXT:    s_and_b32 s17, s6, 15
1819; GFX9-DL-NEXT:    s_bfe_u32 s6, s6, 0x40004
1820; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v4, 12, s2 op_sel_hi:[0,1]
1821; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s17, s6
1822; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v5, 12, s2 op_sel_hi:[0,1]
1823; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
1824; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
1825; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
1826; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v1, v1, v5
1827; GFX9-DL-NEXT:    global_load_ushort v5, v0, s[0:1]
1828; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s15, s16
1829; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s2 op_sel_hi:[0,1]
1830; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s13, s14
1831; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
1832; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
1833; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v7, 12, s2 op_sel_hi:[0,1]
1834; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v6
1835; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s2, s7, s12
1836; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
1837; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
1838; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v8, 12, s2 op_sel_hi:[0,1]
1839; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v7
1840; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
1841; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
1842; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v8
1843; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
1844; GFX9-DL-NEXT:    v_add_u32_e32 v5, v1, v5
1845; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1846; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
1847; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1848; GFX9-DL-NEXT:    v_add_u32_e32 v1, v1, v3
1849; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1850; GFX9-DL-NEXT:    v_add_u32_e32 v1, v1, v4
1851; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1852; GFX9-DL-NEXT:    global_store_short v0, v1, s[0:1]
1853; GFX9-DL-NEXT:    s_endpgm
1854;
1855; GFX10-DL-LABEL: idot8_acc16_vecMul:
1856; GFX10-DL:       ; %bb.0: ; %entry
1857; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1858; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
1859; GFX10-DL-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
1860; GFX10-DL-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
1861; GFX10-DL-NEXT:    s_mov_b32 s14, -1
1862; GFX10-DL-NEXT:    s_mov_b32 s15, 0x31c16000
1863; GFX10-DL-NEXT:    s_add_u32 s12, s12, s3
1864; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1865; GFX10-DL-NEXT:    s_addc_u32 s13, s13, 0
1866; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1867; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1868; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
1869; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
1870; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
1871; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1872; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x40018
1873; GFX10-DL-NEXT:    s_lshr_b32 s3, s0, 28
1874; GFX10-DL-NEXT:    s_bfe_u32 s6, s0, 0x40010
1875; GFX10-DL-NEXT:    s_bfe_u32 s7, s0, 0x40014
1876; GFX10-DL-NEXT:    s_bfe_u32 s8, s0, 0x40008
1877; GFX10-DL-NEXT:    s_bfe_u32 s9, s0, 0x4000c
1878; GFX10-DL-NEXT:    s_and_b32 s10, s0, 15
1879; GFX10-DL-NEXT:    s_bfe_u32 s0, s0, 0x40004
1880; GFX10-DL-NEXT:    s_and_b32 s11, s1, 15
1881; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s10, s0
1882; GFX10-DL-NEXT:    s_bfe_u32 s10, s1, 0x40004
1883; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v2, 12, s0 op_sel_hi:[0,1]
1884; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s11, s10
1885; GFX10-DL-NEXT:    s_bfe_u32 s11, s1, 0x4000c
1886; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
1887; GFX10-DL-NEXT:    s_bfe_u32 s0, s1, 0x40008
1888; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
1889; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
1890; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s0, s11
1891; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
1892; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v4, 12, s8 op_sel_hi:[0,1]
1893; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
1894; GFX10-DL-NEXT:    s_bfe_u32 s8, s1, 0x40010
1895; GFX10-DL-NEXT:    s_bfe_u32 s0, s1, 0x40014
1896; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v3
1897; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v3, 12, v4 op_sel_hi:[0,1]
1898; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v5 op_sel_hi:[0,1]
1899; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
1900; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s8, s0
1901; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v5, 12, s6 op_sel_hi:[0,1]
1902; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
1903; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v4
1904; GFX10-DL-NEXT:    s_bfe_u32 s10, s1, 0x40018
1905; GFX10-DL-NEXT:    s_lshr_b32 s0, s1, 28
1906; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s1, s2, s3
1907; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v6 op_sel_hi:[0,1]
1908; GFX10-DL-NEXT:    s_pack_ll_b32_b16 s0, s10, s0
1909; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
1910; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
1911; GFX10-DL-NEXT:    v_add_nc_u32_e32 v1, v2, v1
1912; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1913; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v2, 12, v5 op_sel_hi:[0,1]
1914; GFX10-DL-NEXT:    v_pk_lshlrev_b16 v5, 12, s1 op_sel_hi:[0,1]
1915; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
1916; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v4
1917; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v6 op_sel_hi:[0,1]
1918; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1919; GFX10-DL-NEXT:    v_pk_ashrrev_i16 v3, 12, v5 op_sel_hi:[0,1]
1920; GFX10-DL-NEXT:    v_add_nc_u32_e32 v1, v1, v2
1921; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v4
1922; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1923; GFX10-DL-NEXT:    v_add_nc_u32_e32 v1, v1, v3
1924; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1925; GFX10-DL-NEXT:    global_store_short v0, v1, s[4:5]
1926; GFX10-DL-NEXT:    s_endpgm
1927                                              <8 x i4> addrspace(1)* %src2,
1928                                              i16 addrspace(1)* nocapture %dst) {
1929entry:
1930  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
1931  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
1932
1933  %cvec1 = sext <8 x i4> %vec1 to <8 x i16>
1934  %cvec2 = sext <8 x i4> %vec2 to <8 x i16>
1935
1936  %mul = mul <8 x i16> %cvec1, %cvec2
1937  %mul0 = extractelement <8 x i16> %mul, i64 0
1938  %mul1 = extractelement <8 x i16> %mul, i64 1
1939  %mul2 = extractelement <8 x i16> %mul, i64 2
1940  %mul3 = extractelement <8 x i16> %mul, i64 3
1941  %mul4 = extractelement <8 x i16> %mul, i64 4
1942  %mul5 = extractelement <8 x i16> %mul, i64 5
1943  %mul6 = extractelement <8 x i16> %mul, i64 6
1944  %mul7 = extractelement <8 x i16> %mul, i64 7
1945
1946  %acc = load i16, i16 addrspace(1)* %dst, align 4
1947  %add1 = add i16 %mul0, %acc
1948  %add2 = add i16 %add1, %mul1
1949  %add3 = add i16 %add2, %mul2
1950  %add4 = add i16 %add3, %mul3
1951  %add5 = add i16 %add4, %mul4
1952  %add6 = add i16 %add5, %mul5
1953  %add7 = add i16 %add6, %mul6
1954  %add8 = add i16 %add7, %mul7
1955
1956  store i16 %add8, i16 addrspace(1)* %dst, align 4
1957  ret void
1958}
1959
1960; TODO: Support this pattern.
1961define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
1962; GFX7-LABEL: idot8_acc8_vecMul:
1963; GFX7:       ; %bb.0: ; %entry
1964; GFX7-NEXT:    s_mov_b32 s24, SCRATCH_RSRC_DWORD0
1965; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1966; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1967; GFX7-NEXT:    s_mov_b32 s25, SCRATCH_RSRC_DWORD1
1968; GFX7-NEXT:    s_mov_b32 s26, -1
1969; GFX7-NEXT:    s_mov_b32 s27, 0xe8f000
1970; GFX7-NEXT:    s_add_u32 s24, s24, s3
1971; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1972; GFX7-NEXT:    s_mov_b32 s2, -1
1973; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1974; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
1975; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1976; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1977; GFX7-NEXT:    s_addc_u32 s25, s25, 0
1978; GFX7-NEXT:    s_movk_i32 s8, 0xff
1979; GFX7-NEXT:    s_mov_b32 s9, 0xffff
1980; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1981; GFX7-NEXT:    s_bfe_i32 s6, s4, 0x40000
1982; GFX7-NEXT:    s_bfe_i32 s15, s5, 0x40000
1983; GFX7-NEXT:    s_bfe_i32 s16, s5, 0x40004
1984; GFX7-NEXT:    s_bfe_i32 s17, s5, 0x40008
1985; GFX7-NEXT:    s_bfe_i32 s18, s5, 0x4000c
1986; GFX7-NEXT:    s_bfe_i32 s19, s5, 0x40010
1987; GFX7-NEXT:    s_bfe_i32 s20, s5, 0x40014
1988; GFX7-NEXT:    s_bfe_i32 s21, s5, 0x40018
1989; GFX7-NEXT:    s_ashr_i32 s5, s5, 28
1990; GFX7-NEXT:    v_mov_b32_e32 v8, s15
1991; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x40004
1992; GFX7-NEXT:    v_mov_b32_e32 v7, s16
1993; GFX7-NEXT:    s_bfe_i32 s10, s4, 0x40008
1994; GFX7-NEXT:    v_mov_b32_e32 v6, s17
1995; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x4000c
1996; GFX7-NEXT:    v_mov_b32_e32 v5, s18
1997; GFX7-NEXT:    s_bfe_i32 s12, s4, 0x40010
1998; GFX7-NEXT:    v_mov_b32_e32 v4, s19
1999; GFX7-NEXT:    s_bfe_i32 s13, s4, 0x40014
2000; GFX7-NEXT:    v_mov_b32_e32 v3, s20
2001; GFX7-NEXT:    s_bfe_i32 s14, s4, 0x40018
2002; GFX7-NEXT:    v_mov_b32_e32 v2, s21
2003; GFX7-NEXT:    s_ashr_i32 s4, s4, 28
2004; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2005; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s4, v1
2006; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s14, v2
2007; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s13, v3
2008; GFX7-NEXT:    v_mul_i32_i24_e32 v9, s12, v4
2009; GFX7-NEXT:    v_mul_i32_i24_e32 v5, s11, v5
2010; GFX7-NEXT:    v_mul_i32_i24_e32 v6, s10, v6
2011; GFX7-NEXT:    v_mul_i32_i24_e32 v7, s7, v7
2012; GFX7-NEXT:    v_mul_i32_i24_e32 v8, s6, v8
2013; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
2014; GFX7-NEXT:    v_and_b32_e32 v2, s8, v2
2015; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
2016; GFX7-NEXT:    v_and_b32_e32 v9, s8, v9
2017; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
2018; GFX7-NEXT:    v_and_b32_e32 v6, s8, v6
2019; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
2020; GFX7-NEXT:    v_and_b32_e32 v8, s8, v8
2021; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
2022; GFX7-NEXT:    v_or_b32_e32 v2, v9, v3
2023; GFX7-NEXT:    v_or_b32_e32 v3, v6, v5
2024; GFX7-NEXT:    v_or_b32_e32 v5, v8, v7
2025; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
2026; GFX7-NEXT:    v_and_b32_e32 v2, s9, v2
2027; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
2028; GFX7-NEXT:    v_and_b32_e32 v5, s9, v5
2029; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
2030; GFX7-NEXT:    v_or_b32_e32 v2, v5, v3
2031; GFX7-NEXT:    v_alignbit_b32 v3, v1, v2, 8
2032; GFX7-NEXT:    v_alignbit_b32 v5, v1, v2, 16
2033; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 24, v2
2034; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 8, v1
2035; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 16, v1
2036; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
2037; GFX7-NEXT:    s_waitcnt vmcnt(0)
2038; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
2039; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
2040; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
2041; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
2042; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v4, v0
2043; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v7
2044; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
2045; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
2046; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
2047; GFX7-NEXT:    s_endpgm
2048;
2049; GFX8-LABEL: idot8_acc8_vecMul:
2050; GFX8:       ; %bb.0: ; %entry
2051; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2052; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2053; GFX8-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
2054; GFX8-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
2055; GFX8-NEXT:    s_mov_b32 s22, -1
2056; GFX8-NEXT:    s_mov_b32 s23, 0xe80000
2057; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2058; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2059; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2060; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
2061; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
2062; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
2063; GFX8-NEXT:    s_add_u32 s20, s20, s3
2064; GFX8-NEXT:    s_addc_u32 s21, s21, 0
2065; GFX8-NEXT:    s_mov_b32 s0, 0xffff
2066; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2067; GFX8-NEXT:    s_bfe_i32 s7, s1, 0x40004
2068; GFX8-NEXT:    s_bfe_i32 s9, s1, 0x4000c
2069; GFX8-NEXT:    s_bfe_i32 s14, s2, 0x40004
2070; GFX8-NEXT:    s_bfe_i32 s15, s2, 0x40000
2071; GFX8-NEXT:    s_bfe_i32 s16, s2, 0x4000c
2072; GFX8-NEXT:    s_bfe_i32 s3, s1, 0x40014
2073; GFX8-NEXT:    s_ashr_i32 s5, s1, 28
2074; GFX8-NEXT:    s_bfe_i32 s10, s2, 0x40014
2075; GFX8-NEXT:    s_bfe_i32 s11, s2, 0x40010
2076; GFX8-NEXT:    s_ashr_i32 s12, s2, 28
2077; GFX8-NEXT:    s_bfe_i32 s13, s2, 0x40018
2078; GFX8-NEXT:    s_bfe_i32 s2, s2, 0x40008
2079; GFX8-NEXT:    s_bfe_i32 s8, s1, 0x40000
2080; GFX8-NEXT:    v_mov_b32_e32 v4, s16
2081; GFX8-NEXT:    v_mov_b32_e32 v5, s9
2082; GFX8-NEXT:    v_mov_b32_e32 v6, s15
2083; GFX8-NEXT:    v_mov_b32_e32 v7, s14
2084; GFX8-NEXT:    v_mov_b32_e32 v8, s7
2085; GFX8-NEXT:    v_mul_i32_i24_sdwa v4, v5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2086; GFX8-NEXT:    v_mul_i32_i24_e32 v5, s8, v6
2087; GFX8-NEXT:    v_mul_i32_i24_sdwa v6, v8, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2088; GFX8-NEXT:    s_bfe_i32 s4, s1, 0x40010
2089; GFX8-NEXT:    s_bfe_i32 s6, s1, 0x40018
2090; GFX8-NEXT:    v_mov_b32_e32 v9, s13
2091; GFX8-NEXT:    s_bfe_i32 s1, s1, 0x40008
2092; GFX8-NEXT:    v_mov_b32_e32 v3, s2
2093; GFX8-NEXT:    v_mov_b32_e32 v10, s12
2094; GFX8-NEXT:    v_mov_b32_e32 v11, s5
2095; GFX8-NEXT:    v_mov_b32_e32 v12, s11
2096; GFX8-NEXT:    v_mov_b32_e32 v13, s10
2097; GFX8-NEXT:    v_mov_b32_e32 v14, s3
2098; GFX8-NEXT:    v_mul_i32_i24_e32 v3, s1, v3
2099; GFX8-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2100; GFX8-NEXT:    v_mul_i32_i24_e32 v7, s6, v9
2101; GFX8-NEXT:    v_mul_i32_i24_sdwa v8, v11, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2102; GFX8-NEXT:    v_mul_i32_i24_e32 v9, s4, v12
2103; GFX8-NEXT:    v_mul_i32_i24_sdwa v10, v14, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2104; GFX8-NEXT:    v_and_b32_e32 v5, s0, v5
2105; GFX8-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2106; GFX8-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2107; GFX8-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2108; GFX8-NEXT:    v_and_b32_e32 v4, s0, v9
2109; GFX8-NEXT:    v_or_b32_e32 v3, v5, v3
2110; GFX8-NEXT:    v_or_b32_e32 v6, v4, v7
2111; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 8, v3
2112; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 8, v6
2113; GFX8-NEXT:    s_waitcnt vmcnt(0)
2114; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
2115; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v7, v2
2116; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0
2117; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
2118; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
2119; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v8, v2
2120; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
2121; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
2122; GFX8-NEXT:    flat_store_byte v[0:1], v2
2123; GFX8-NEXT:    s_endpgm
2124;
2125; GFX9-LABEL: idot8_acc8_vecMul:
2126; GFX9:       ; %bb.0: ; %entry
2127; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2128; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2129; GFX9-NEXT:    v_mov_b32_e32 v0, 0
2130; GFX9-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
2131; GFX9-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
2132; GFX9-NEXT:    s_mov_b32 s22, -1
2133; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2134; GFX9-NEXT:    global_load_ubyte v1, v0, s[0:1]
2135; GFX9-NEXT:    s_mov_b32 s23, 0xe00000
2136; GFX9-NEXT:    s_add_u32 s20, s20, s3
2137; GFX9-NEXT:    s_load_dword s3, s[4:5], 0x0
2138; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
2139; GFX9-NEXT:    s_addc_u32 s21, s21, 0
2140; GFX9-NEXT:    s_mov_b32 s2, 0xffff
2141; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2142; GFX9-NEXT:    s_lshr_b32 s9, s3, 4
2143; GFX9-NEXT:    s_lshr_b32 s16, s4, 4
2144; GFX9-NEXT:    v_lshlrev_b16_e64 v2, 12, s3
2145; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 12, s4
2146; GFX9-NEXT:    v_lshlrev_b16_e64 v6, 12, s9
2147; GFX9-NEXT:    v_lshlrev_b16_e64 v13, 12, s16
2148; GFX9-NEXT:    s_lshr_b32 s10, s3, 12
2149; GFX9-NEXT:    s_lshr_b32 s11, s3, 8
2150; GFX9-NEXT:    s_lshr_b32 s17, s4, 12
2151; GFX9-NEXT:    s_lshr_b32 s18, s4, 8
2152; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s11
2153; GFX9-NEXT:    v_lshlrev_b16_e64 v5, 12, s10
2154; GFX9-NEXT:    v_lshlrev_b16_e64 v11, 12, s18
2155; GFX9-NEXT:    v_lshlrev_b16_e64 v12, 12, s17
2156; GFX9-NEXT:    v_ashrrev_i16_e32 v2, 12, v2
2157; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
2158; GFX9-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
2159; GFX9-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
2160; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
2161; GFX9-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
2162; GFX9-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
2163; GFX9-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
2164; GFX9-NEXT:    v_mul_lo_u16_e32 v2, v2, v3
2165; GFX9-NEXT:    v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2166; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2167; GFX9-NEXT:    s_lshr_b32 s5, s3, 20
2168; GFX9-NEXT:    s_lshr_b32 s6, s3, 16
2169; GFX9-NEXT:    s_lshr_b32 s12, s4, 20
2170; GFX9-NEXT:    s_lshr_b32 s13, s4, 16
2171; GFX9-NEXT:    v_mul_lo_u16_sdwa v5, v5, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2172; GFX9-NEXT:    v_mul_lo_u16_e32 v4, v4, v11
2173; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
2174; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
2175; GFX9-NEXT:    v_lshlrev_b16_e64 v16, 12, s13
2176; GFX9-NEXT:    v_lshlrev_b16_e64 v17, 12, s12
2177; GFX9-NEXT:    s_lshr_b32 s7, s3, 28
2178; GFX9-NEXT:    s_lshr_b32 s8, s3, 24
2179; GFX9-NEXT:    s_lshr_b32 s14, s4, 28
2180; GFX9-NEXT:    s_lshr_b32 s15, s4, 24
2181; GFX9-NEXT:    v_and_b32_e32 v2, s2, v2
2182; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2183; GFX9-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
2184; GFX9-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
2185; GFX9-NEXT:    v_lshlrev_b16_e64 v14, 12, s15
2186; GFX9-NEXT:    v_lshlrev_b16_e64 v15, 12, s14
2187; GFX9-NEXT:    v_or_b32_e32 v4, v2, v4
2188; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
2189; GFX9-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
2190; GFX9-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
2191; GFX9-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
2192; GFX9-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
2193; GFX9-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
2194; GFX9-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
2195; GFX9-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
2196; GFX9-NEXT:    v_mul_lo_u16_sdwa v3, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2197; GFX9-NEXT:    v_mul_lo_u16_e32 v9, v9, v16
2198; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 8, v4
2199; GFX9-NEXT:    v_or_b32_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2200; GFX9-NEXT:    v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2201; GFX9-NEXT:    v_mul_lo_u16_e32 v7, v7, v14
2202; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2203; GFX9-NEXT:    v_and_b32_e32 v3, s2, v3
2204; GFX9-NEXT:    v_or_b32_e32 v5, v3, v7
2205; GFX9-NEXT:    s_waitcnt vmcnt(0)
2206; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
2207; GFX9-NEXT:    v_add_u32_e32 v1, v1, v6
2208; GFX9-NEXT:    v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
2209; GFX9-NEXT:    v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2210; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
2211; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 8, v5
2212; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
2213; GFX9-NEXT:    v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2214; GFX9-NEXT:    v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2215; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
2216; GFX9-NEXT:    s_endpgm
2217;
2218; GFX9-DL-LABEL: idot8_acc8_vecMul:
2219; GFX9-DL:       ; %bb.0: ; %entry
2220; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2221; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2222; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
2223; GFX9-DL-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
2224; GFX9-DL-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
2225; GFX9-DL-NEXT:    s_mov_b32 s22, -1
2226; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2227; GFX9-DL-NEXT:    global_load_ubyte v1, v0, s[0:1]
2228; GFX9-DL-NEXT:    s_mov_b32 s23, 0xe00000
2229; GFX9-DL-NEXT:    s_add_u32 s20, s20, s3
2230; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
2231; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
2232; GFX9-DL-NEXT:    s_addc_u32 s21, s21, 0
2233; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
2234; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2235; GFX9-DL-NEXT:    s_lshr_b32 s9, s3, 4
2236; GFX9-DL-NEXT:    s_lshr_b32 s16, s4, 4
2237; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v2, 12, s3
2238; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s4
2239; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s9
2240; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s16
2241; GFX9-DL-NEXT:    s_lshr_b32 s10, s3, 12
2242; GFX9-DL-NEXT:    s_lshr_b32 s11, s3, 8
2243; GFX9-DL-NEXT:    s_lshr_b32 s17, s4, 12
2244; GFX9-DL-NEXT:    s_lshr_b32 s18, s4, 8
2245; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s11
2246; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s10
2247; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s18
2248; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s17
2249; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v2, 12, v2
2250; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
2251; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
2252; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
2253; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
2254; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
2255; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
2256; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
2257; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v2, v2, v3
2258; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2259; GFX9-DL-NEXT:    v_or_b32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2260; GFX9-DL-NEXT:    s_lshr_b32 s5, s3, 20
2261; GFX9-DL-NEXT:    s_lshr_b32 s6, s3, 16
2262; GFX9-DL-NEXT:    s_lshr_b32 s12, s4, 20
2263; GFX9-DL-NEXT:    s_lshr_b32 s13, s4, 16
2264; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v5, v5, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2265; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v4, v4, v11
2266; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
2267; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
2268; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v16, 12, s13
2269; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v17, 12, s12
2270; GFX9-DL-NEXT:    s_lshr_b32 s7, s3, 28
2271; GFX9-DL-NEXT:    s_lshr_b32 s8, s3, 24
2272; GFX9-DL-NEXT:    s_lshr_b32 s14, s4, 28
2273; GFX9-DL-NEXT:    s_lshr_b32 s15, s4, 24
2274; GFX9-DL-NEXT:    v_and_b32_e32 v2, s2, v2
2275; GFX9-DL-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2276; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
2277; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
2278; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v14, 12, s15
2279; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s14
2280; GFX9-DL-NEXT:    v_or_b32_e32 v4, v2, v4
2281; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
2282; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
2283; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
2284; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
2285; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
2286; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
2287; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
2288; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
2289; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v3, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2290; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v9, v9, v16
2291; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v6, 8, v4
2292; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2293; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2294; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v7, v7, v14
2295; GFX9-DL-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2296; GFX9-DL-NEXT:    v_and_b32_e32 v3, s2, v3
2297; GFX9-DL-NEXT:    v_or_b32_e32 v5, v3, v7
2298; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2299; GFX9-DL-NEXT:    v_add_u32_e32 v1, v2, v1
2300; GFX9-DL-NEXT:    v_add_u32_e32 v1, v1, v6
2301; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
2302; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2303; GFX9-DL-NEXT:    v_add_u32_e32 v1, v1, v3
2304; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v2, 8, v5
2305; GFX9-DL-NEXT:    v_add_u32_e32 v1, v1, v2
2306; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2307; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2308; GFX9-DL-NEXT:    global_store_byte v0, v1, s[0:1]
2309; GFX9-DL-NEXT:    s_endpgm
2310;
2311; GFX10-DL-LABEL: idot8_acc8_vecMul:
2312; GFX10-DL:       ; %bb.0: ; %entry
2313; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
2314; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
2315; GFX10-DL-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
2316; GFX10-DL-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
2317; GFX10-DL-NEXT:    s_mov_b32 s22, -1
2318; GFX10-DL-NEXT:    s_mov_b32 s23, 0x31c16000
2319; GFX10-DL-NEXT:    s_add_u32 s20, s20, s3
2320; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2321; GFX10-DL-NEXT:    s_addc_u32 s21, s21, 0
2322; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2323; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2324; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
2325; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
2326; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
2327; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
2328; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2329; GFX10-DL-NEXT:    s_lshr_b32 s9, s0, 4
2330; GFX10-DL-NEXT:    s_lshr_b32 s16, s1, 4
2331; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s9
2332; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s16
2333; GFX10-DL-NEXT:    s_lshr_b32 s10, s0, 12
2334; GFX10-DL-NEXT:    s_lshr_b32 s17, s1, 12
2335; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v2, 12, s0
2336; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s1
2337; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s17
2338; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s10
2339; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v6, 12, v6
2340; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v12, 12, v12
2341; GFX10-DL-NEXT:    s_lshr_b32 s11, s0, 8
2342; GFX10-DL-NEXT:    s_lshr_b32 s18, s1, 8
2343; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s11
2344; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s18
2345; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v2, 12, v2
2346; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v3, 12, v3
2347; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v19, 12, v5
2348; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v6, v6, v12
2349; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v13, 12, v13
2350; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v4
2351; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v11, 12, v11
2352; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v2, v2, v3
2353; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 8, v6
2354; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v3, v19, v13
2355; GFX10-DL-NEXT:    s_lshr_b32 s3, s0, 20
2356; GFX10-DL-NEXT:    s_lshr_b32 s6, s0, 16
2357; GFX10-DL-NEXT:    s_lshr_b32 s7, s0, 28
2358; GFX10-DL-NEXT:    s_lshr_b32 s8, s0, 24
2359; GFX10-DL-NEXT:    s_lshr_b32 s12, s1, 20
2360; GFX10-DL-NEXT:    v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2361; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
2362; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
2363; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
2364; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s3
2365; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s12
2366; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, v4, v11
2367; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v3
2368; GFX10-DL-NEXT:    s_lshr_b32 s13, s1, 16
2369; GFX10-DL-NEXT:    s_lshr_b32 s14, s1, 28
2370; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s13
2371; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 12, v7
2372; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v7, 12, v8
2373; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v8, 12, v9
2374; GFX10-DL-NEXT:    v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2375; GFX10-DL-NEXT:    v_and_b32_e32 v2, s2, v2
2376; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s14
2377; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 12, v10
2378; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v9, 12, v12
2379; GFX10-DL-NEXT:    s_lshr_b32 s15, s1, 24
2380; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v6, 12, v6
2381; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v10, 12, v15
2382; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v14, 12, s15
2383; GFX10-DL-NEXT:    v_or_b32_e32 v3, v2, v3
2384; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, v4, v9
2385; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v15, v8, v6
2386; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, v7, v10
2387; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v11, 12, v14
2388; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v8, 8, v3
2389; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
2390; GFX10-DL-NEXT:    v_add_nc_u32_e32 v1, v2, v1
2391; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v2, 8, v4
2392; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, v5, v11
2393; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v5, 8, v7
2394; GFX10-DL-NEXT:    v_add_nc_u32_e32 v1, v1, v8
2395; GFX10-DL-NEXT:    v_or_b32_sdwa v2, v15, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2396; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2397; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
2398; GFX10-DL-NEXT:    v_and_b32_e32 v2, s2, v2
2399; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2400; GFX10-DL-NEXT:    v_or_b32_e32 v3, v2, v4
2401; GFX10-DL-NEXT:    v_add_nc_u32_e32 v1, v1, v2
2402; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v2, 8, v3
2403; GFX10-DL-NEXT:    v_add_nc_u32_e32 v1, v1, v2
2404; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2405; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2406; GFX10-DL-NEXT:    global_store_byte v0, v1, s[4:5]
2407; GFX10-DL-NEXT:    s_endpgm
2408                                             <8 x i4> addrspace(1)* %src2,
2409                                             i8 addrspace(1)* nocapture %dst) {
2410entry:
2411  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
2412  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
2413
2414  %cvec1 = sext <8 x i4> %vec1 to <8 x i8>
2415  %cvec2 = sext <8 x i4> %vec2 to <8 x i8>
2416
2417  %mul = mul <8 x i8> %cvec1, %cvec2
2418  %mul0 = extractelement <8 x i8> %mul, i64 0
2419  %mul1 = extractelement <8 x i8> %mul, i64 1
2420  %mul2 = extractelement <8 x i8> %mul, i64 2
2421  %mul3 = extractelement <8 x i8> %mul, i64 3
2422  %mul4 = extractelement <8 x i8> %mul, i64 4
2423  %mul5 = extractelement <8 x i8> %mul, i64 5
2424  %mul6 = extractelement <8 x i8> %mul, i64 6
2425  %mul7 = extractelement <8 x i8> %mul, i64 7
2426
2427  %acc = load i8, i8 addrspace(1)* %dst, align 4
2428  %add1 = add i8 %mul0, %acc
2429  %add2 = add i8 %add1, %mul1
2430  %add3 = add i8 %add2, %mul2
2431  %add4 = add i8 %add3, %mul3
2432  %add5 = add i8 %add4, %mul4
2433  %add6 = add i8 %add5, %mul5
2434  %add7 = add i8 %add6, %mul6
2435  %add8 = add i8 %add7, %mul7
2436
2437  store i8 %add8, i8 addrspace(1)* %dst, align 4
2438  ret void
2439}
2440