1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-NODL %s
5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
8
9define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
10; GFX7-LABEL: idot4_acc32:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
13; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
14; GFX7-NEXT:    s_mov_b32 s3, 0xf000
15; GFX7-NEXT:    s_mov_b32 s2, -1
16; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
17; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
18; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
19; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
20; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX7-NEXT:    s_sext_i32_i8 s6, s4
22; GFX7-NEXT:    s_sext_i32_i8 s7, s5
23; GFX7-NEXT:    s_bfe_i32 s9, s5, 0x80008
24; GFX7-NEXT:    v_mov_b32_e32 v0, s7
25; GFX7-NEXT:    v_mov_b32_e32 v1, s12
26; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80010
27; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v0, v1
28; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80008
29; GFX7-NEXT:    v_mov_b32_e32 v1, s9
30; GFX7-NEXT:    s_bfe_i32 s10, s4, 0x80010
31; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
32; GFX7-NEXT:    v_mov_b32_e32 v1, s11
33; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
34; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
35; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
36; GFX7-NEXT:    v_mov_b32_e32 v1, s5
37; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
38; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
39; GFX7-NEXT:    s_endpgm
40;
41; GFX8-LABEL: idot4_acc32:
42; GFX8:       ; %bb.0: ; %entry
43; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
44; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
45; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
46; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
47; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
48; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
49; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
50; GFX8-NEXT:    s_sext_i32_i8 s4, s2
51; GFX8-NEXT:    s_sext_i32_i8 s5, s3
52; GFX8-NEXT:    s_bfe_i32 s7, s3, 0x80008
53; GFX8-NEXT:    v_mov_b32_e32 v0, s5
54; GFX8-NEXT:    v_mov_b32_e32 v1, s10
55; GFX8-NEXT:    s_bfe_i32 s9, s3, 0x80010
56; GFX8-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
57; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x80008
58; GFX8-NEXT:    v_mov_b32_e32 v1, s7
59; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x80010
60; GFX8-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
61; GFX8-NEXT:    v_mov_b32_e32 v1, s9
62; GFX8-NEXT:    s_ashr_i32 s3, s3, 24
63; GFX8-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
64; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
65; GFX8-NEXT:    v_mov_b32_e32 v1, s3
66; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
67; GFX8-NEXT:    v_mov_b32_e32 v0, s0
68; GFX8-NEXT:    v_mov_b32_e32 v1, s1
69; GFX8-NEXT:    flat_store_dword v[0:1], v2
70; GFX8-NEXT:    s_endpgm
71;
72; GFX9-NODL-LABEL: idot4_acc32:
73; GFX9-NODL:       ; %bb.0: ; %entry
74; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
75; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
76; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
77; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
78; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
79; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
80; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
81; GFX9-NODL-NEXT:    s_sext_i32_i8 s4, s2
82; GFX9-NODL-NEXT:    s_sext_i32_i8 s5, s3
83; GFX9-NODL-NEXT:    s_bfe_i32 s7, s3, 0x80008
84; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
85; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s10
86; GFX9-NODL-NEXT:    s_bfe_i32 s9, s3, 0x80010
87; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
88; GFX9-NODL-NEXT:    s_bfe_i32 s6, s2, 0x80008
89; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s7
90; GFX9-NODL-NEXT:    s_bfe_i32 s8, s2, 0x80010
91; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
92; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s9
93; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 24
94; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
95; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
96; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
97; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
98; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
99; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
100; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
101; GFX9-NODL-NEXT:    s_endpgm
102;
103; GFX9-DL-LABEL: idot4_acc32:
104; GFX9-DL:       ; %bb.0: ; %entry
105; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
106; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
107; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
108; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
109; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
110; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
111; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
112; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
113; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
114; GFX9-DL-NEXT:    v_dot4_i32_i8 v2, s4, v0, v1
115; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
116; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
117; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
118; GFX9-DL-NEXT:    s_endpgm
119;
120; GFX10-DL-LABEL: idot4_acc32:
121; GFX10-DL:       ; %bb.0: ; %entry
122; GFX10-DL-NEXT:    s_clause 0x1
123; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
124; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
125; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
126; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
127; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
128; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
129; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
130; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
131; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
132; GFX10-DL-NEXT:    v_dot4_i32_i8 v2, s0, s1, v0
133; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
134; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
135; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
136; GFX10-DL-NEXT:    s_endpgm
137                                       <4 x i8> addrspace(1)* %src2,
138                                       i32 addrspace(1)* nocapture %dst) {
139entry:
140  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
141  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
142
143  %v1e0 = extractelement <4 x i8> %vec1, i64 0
144  %cv1e0 = sext i8 %v1e0 to i32
145  %v2e0 = extractelement <4 x i8> %vec2, i64 0
146  %cv2e0 = sext i8 %v2e0 to i32
147  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
148
149  %v1e1 = extractelement <4 x i8> %vec1, i64 1
150  %cv1e1 = sext i8 %v1e1 to i32
151  %v2e1 = extractelement <4 x i8> %vec2, i64 1
152  %cv2e1 = sext i8 %v2e1 to i32
153  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
154
155  %v1e2 = extractelement <4 x i8> %vec1, i64 2
156  %cv1e2 = sext i8 %v1e2 to i32
157  %v2e2 = extractelement <4 x i8> %vec2, i64 2
158  %cv2e2 = sext i8 %v2e2 to i32
159  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
160
161  %v1e3 = extractelement <4 x i8> %vec1, i64 3
162  %cv1e3 = sext i8 %v1e3 to i32
163  %v2e3 = extractelement <4 x i8> %vec2, i64 3
164  %cv2e3 = sext i8 %v2e3 to i32
165  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
166
167  %acc = load i32, i32 addrspace(1)* %dst, align 4
168  %add1 = add i32 %mul1, %acc
169  %add2 = add i32 %add1, %mul2
170  %add3 = add i32 %add2, %mul3
171  %add4 = add i32 %add3, %mul4
172  store i32 %add4, i32 addrspace(1)* %dst, align 4
173  ret void
174}
175
176; TODO: Currently, vector elements{0 and 3} get zero_extended from i16 to i32 which should
177; be sign_extended directly to i32; prevents the pattern recognizer to recognize this pattern.
178define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
179; GFX7-LABEL: idot4_acc16:
180; GFX7:       ; %bb.0: ; %entry
181; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
182; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
183; GFX7-NEXT:    s_mov_b32 s3, 0xf000
184; GFX7-NEXT:    s_mov_b32 s2, -1
185; GFX7-NEXT:    s_mov_b32 s8, 0xffff
186; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
187; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
188; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
189; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
190; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
191; GFX7-NEXT:    s_sext_i32_i8 s6, s4
192; GFX7-NEXT:    s_sext_i32_i8 s7, s5
193; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80008
194; GFX7-NEXT:    s_and_b32 s7, s7, s8
195; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
196; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
197; GFX7-NEXT:    s_and_b32 s10, s10, s8
198; GFX7-NEXT:    s_and_b32 s6, s6, s8
199; GFX7-NEXT:    v_mov_b32_e32 v1, s7
200; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x80010
201; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
202; GFX7-NEXT:    s_and_b32 s12, s12, s8
203; GFX7-NEXT:    s_and_b32 s9, s9, s8
204; GFX7-NEXT:    v_mov_b32_e32 v2, s10
205; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
206; GFX7-NEXT:    s_and_b32 s11, s11, s8
207; GFX7-NEXT:    s_and_b32 s5, s5, s8
208; GFX7-NEXT:    v_mov_b32_e32 v3, s12
209; GFX7-NEXT:    s_and_b32 s4, s4, s8
210; GFX7-NEXT:    s_waitcnt vmcnt(0)
211; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
212; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
213; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
214; GFX7-NEXT:    v_mov_b32_e32 v1, s5
215; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
216; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
217; GFX7-NEXT:    s_endpgm
218;
219; GFX8-LABEL: idot4_acc16:
220; GFX8:       ; %bb.0: ; %entry
221; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
222; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
223; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
224; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
225; GFX8-NEXT:    v_mov_b32_e32 v0, s0
226; GFX8-NEXT:    v_mov_b32_e32 v1, s1
227; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
228; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
229; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
230; GFX8-NEXT:    s_sext_i32_i8 s3, s2
231; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x80008
232; GFX8-NEXT:    v_mov_b32_e32 v3, s3
233; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x80010
234; GFX8-NEXT:    s_sext_i32_i8 s1, s0
235; GFX8-NEXT:    s_bfe_i32 s4, s0, 0x80008
236; GFX8-NEXT:    v_mov_b32_e32 v4, s5
237; GFX8-NEXT:    s_bfe_i32 s6, s0, 0x80010
238; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
239; GFX8-NEXT:    v_mov_b32_e32 v5, s7
240; GFX8-NEXT:    s_ashr_i32 s0, s0, 24
241; GFX8-NEXT:    s_waitcnt vmcnt(0)
242; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
243; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v4, v2
244; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v5, v2
245; GFX8-NEXT:    v_mov_b32_e32 v3, s2
246; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
247; GFX8-NEXT:    flat_store_short v[0:1], v2
248; GFX8-NEXT:    s_endpgm
249;
250; GFX9-NODL-LABEL: idot4_acc16:
251; GFX9-NODL:       ; %bb.0: ; %entry
252; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
253; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
254; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
255; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
256; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
257; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
258; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
259; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
260; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
261; GFX9-NODL-NEXT:    s_sext_i32_i8 s3, s2
262; GFX9-NODL-NEXT:    s_bfe_i32 s5, s2, 0x80008
263; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
264; GFX9-NODL-NEXT:    s_bfe_i32 s7, s2, 0x80010
265; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s0
266; GFX9-NODL-NEXT:    s_bfe_i32 s4, s0, 0x80008
267; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s5
268; GFX9-NODL-NEXT:    s_bfe_i32 s6, s0, 0x80010
269; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
270; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s7
271; GFX9-NODL-NEXT:    s_ashr_i32 s0, s0, 24
272; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
273; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
274; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s4, v4, v2
275; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v5, v2
276; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
277; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
278; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
279; GFX9-NODL-NEXT:    s_endpgm
280;
281; GFX9-DL-LABEL: idot4_acc16:
282; GFX9-DL:       ; %bb.0: ; %entry
283; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
284; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
285; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
286; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
287; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
288; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
289; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
290; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
291; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
292; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
293; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
294; GFX9-DL-NEXT:    v_dot4_i32_i8 v2, s2, v3, v2
295; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
296; GFX9-DL-NEXT:    s_endpgm
297;
298; GFX10-DL-LABEL: idot4_acc16:
299; GFX10-DL:       ; %bb.0: ; %entry
300; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
301; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
302; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
303; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
304; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s3
305; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
306; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
307; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
308; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
309; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
310; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
311; GFX10-DL-NEXT:    v_dot4_i32_i8 v2, s0, s1, v2
312; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
313; GFX10-DL-NEXT:    s_endpgm
314                                       <4 x i8> addrspace(1)* %src2,
315                                       i16 addrspace(1)* nocapture %dst) {
316entry:
317  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
318  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
319
320  %v1e0 = extractelement <4 x i8> %vec1, i64 0
321  %cv1e0 = sext i8 %v1e0 to i16
322  %v2e0 = extractelement <4 x i8> %vec2, i64 0
323  %cv2e0 = sext i8 %v2e0 to i16
324  %mul1 = mul nsw i16 %cv1e0, %cv2e0
325
326  %v1e1 = extractelement <4 x i8> %vec1, i64 1
327  %cv1e1 = sext i8 %v1e1 to i16
328  %v2e1 = extractelement <4 x i8> %vec2, i64 1
329  %cv2e1 = sext i8 %v2e1 to i16
330  %mul2 = mul nsw i16 %cv1e1, %cv2e1
331
332  %v1e2 = extractelement <4 x i8> %vec1, i64 2
333  %cv1e2 = sext i8 %v1e2 to i16
334  %v2e2 = extractelement <4 x i8> %vec2, i64 2
335  %cv2e2 = sext i8 %v2e2 to i16
336  %mul3 = mul nsw i16 %cv1e2, %cv2e2
337
338  %v1e3 = extractelement <4 x i8> %vec1, i64 3
339  %cv1e3 = sext i8 %v1e3 to i16
340  %v2e3 = extractelement <4 x i8> %vec2, i64 3
341  %cv2e3 = sext i8 %v2e3 to i16
342  %mul4 = mul nsw i16 %cv1e3, %cv2e3
343
344  %acc = load i16, i16 addrspace(1)* %dst, align 2
345  %add1 = add i16 %mul1, %acc
346  %add2 = add i16 %add1, %mul2
347  %add3 = add i16 %add2, %mul3
348  %add4 = add i16 %add3, %mul4
349  store i16 %add4, i16 addrspace(1)* %dst, align 2
350  ret void
351}
352
353define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
354; GFX7-LABEL: idot4_acc8:
355; GFX7:       ; %bb.0: ; %entry
356; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
357; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
358; GFX7-NEXT:    s_mov_b32 s3, 0xf000
359; GFX7-NEXT:    s_mov_b32 s2, -1
360; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
361; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
362; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
363; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
364; GFX7-NEXT:    s_movk_i32 s5, 0xff
365; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
366; GFX7-NEXT:    s_and_b32 s7, s6, s5
367; GFX7-NEXT:    s_bfe_u32 s8, s6, 0x80008
368; GFX7-NEXT:    s_and_b32 s5, s4, s5
369; GFX7-NEXT:    v_mov_b32_e32 v1, s7
370; GFX7-NEXT:    s_bfe_u32 s10, s6, 0x80010
371; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
372; GFX7-NEXT:    v_mov_b32_e32 v2, s8
373; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
374; GFX7-NEXT:    s_lshr_b32 s6, s6, 24
375; GFX7-NEXT:    v_mov_b32_e32 v3, s10
376; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
377; GFX7-NEXT:    s_waitcnt vmcnt(0)
378; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
379; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
380; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
381; GFX7-NEXT:    v_mov_b32_e32 v1, s6
382; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
383; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
384; GFX7-NEXT:    s_endpgm
385;
386; GFX8-LABEL: idot4_acc8:
387; GFX8:       ; %bb.0: ; %entry
388; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
389; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
390; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
391; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
392; GFX8-NEXT:    v_mov_b32_e32 v0, s0
393; GFX8-NEXT:    v_mov_b32_e32 v1, s1
394; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
395; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
396; GFX8-NEXT:    s_movk_i32 s0, 0xff
397; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
398; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
399; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
400; GFX8-NEXT:    s_and_b32 s3, s1, s0
401; GFX8-NEXT:    s_and_b32 s0, s2, s0
402; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x80008
403; GFX8-NEXT:    v_mov_b32_e32 v3, s3
404; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
405; GFX8-NEXT:    v_mov_b32_e32 v4, s4
406; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
407; GFX8-NEXT:    v_mov_b32_e32 v5, s6
408; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
409; GFX8-NEXT:    s_waitcnt vmcnt(0)
410; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
411; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
412; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
413; GFX8-NEXT:    v_mov_b32_e32 v3, s1
414; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
415; GFX8-NEXT:    flat_store_byte v[0:1], v2
416; GFX8-NEXT:    s_endpgm
417;
418; GFX9-NODL-LABEL: idot4_acc8:
419; GFX9-NODL:       ; %bb.0: ; %entry
420; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
421; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
422; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
423; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
424; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
425; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
426; GFX9-NODL-NEXT:    global_load_ubyte v2, v[0:1], off
427; GFX9-NODL-NEXT:    s_load_dword s1, s[6:7], 0x0
428; GFX9-NODL-NEXT:    s_movk_i32 s0, 0xff
429; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
430; GFX9-NODL-NEXT:    s_bfe_u32 s5, s2, 0x80008
431; GFX9-NODL-NEXT:    s_bfe_u32 s7, s2, 0x80010
432; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s0
433; GFX9-NODL-NEXT:    s_and_b32 s0, s2, s0
434; GFX9-NODL-NEXT:    s_bfe_u32 s4, s1, 0x80008
435; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
436; GFX9-NODL-NEXT:    s_bfe_u32 s6, s1, 0x80010
437; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s4
438; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 24
439; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s6
440; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
441; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
442; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
443; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
444; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
445; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
446; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
447; GFX9-NODL-NEXT:    global_store_byte v[0:1], v2, off
448; GFX9-NODL-NEXT:    s_endpgm
449;
450; GFX9-DL-LABEL: idot4_acc8:
451; GFX9-DL:       ; %bb.0: ; %entry
452; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
453; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
454; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
455; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
456; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
457; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
458; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
459; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
460; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
461; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
462; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
463; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s2, v3, v2
464; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
465; GFX9-DL-NEXT:    s_endpgm
466;
467; GFX10-DL-LABEL: idot4_acc8:
468; GFX10-DL:       ; %bb.0: ; %entry
469; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
470; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
471; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
472; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
473; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s3
474; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
475; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
476; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
477; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
478; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
479; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
480; GFX10-DL-NEXT:    v_dot4_u32_u8 v2, s0, s1, v2
481; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
482; GFX10-DL-NEXT:    s_endpgm
483                                      <4 x i8> addrspace(1)* %src2,
484                                      i8 addrspace(1)* nocapture %dst) {
485entry:
486  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
487  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
488
489  %v1e0 = extractelement <4 x i8> %vec1, i64 0
490  %v2e0 = extractelement <4 x i8> %vec2, i64 0
491  %mul1 = mul i8 %v1e0, %v2e0
492
493  %v1e1 = extractelement <4 x i8> %vec1, i64 1
494  %v2e1 = extractelement <4 x i8> %vec2, i64 1
495  %mul2 = mul i8 %v1e1, %v2e1
496
497  %v1e2 = extractelement <4 x i8> %vec1, i64 2
498  %v2e2 = extractelement <4 x i8> %vec2, i64 2
499  %mul3 = mul i8 %v1e2, %v2e2
500
501  %v1e3 = extractelement <4 x i8> %vec1, i64 3
502  %v2e3 = extractelement <4 x i8> %vec2, i64 3
503  %mul4 = mul i8 %v1e3, %v2e3
504
505  %acc = load i8, i8 addrspace(1)* %dst, align 2
506  %add1 = add i8 %mul1, %acc
507  %add2 = add i8 %add1, %mul2
508  %add3 = add i8 %add2, %mul3
509  %add4 = add nsw i8 %add3, %mul4
510  store i8 %add4, i8 addrspace(1)* %dst, align 2
511  ret void
512}
513
514define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
515; GFX7-LABEL: idot4_multiuse_mul1:
516; GFX7:       ; %bb.0: ; %entry
517; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
518; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
519; GFX7-NEXT:    s_mov_b32 s3, 0xf000
520; GFX7-NEXT:    s_mov_b32 s2, -1
521; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
522; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
523; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
524; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
525; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
526; GFX7-NEXT:    s_sext_i32_i8 s6, s4
527; GFX7-NEXT:    s_sext_i32_i8 s7, s5
528; GFX7-NEXT:    s_bfe_i32 s9, s5, 0x80008
529; GFX7-NEXT:    v_mov_b32_e32 v0, s7
530; GFX7-NEXT:    v_mov_b32_e32 v1, s12
531; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80008
532; GFX7-NEXT:    v_mad_i32_i24 v1, s6, v0, v1
533; GFX7-NEXT:    v_mov_b32_e32 v2, s9
534; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80010
535; GFX7-NEXT:    v_mad_i32_i24 v1, s8, v2, v1
536; GFX7-NEXT:    s_bfe_i32 s10, s4, 0x80010
537; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v0, v1
538; GFX7-NEXT:    v_mov_b32_e32 v1, s11
539; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
540; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
541; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
542; GFX7-NEXT:    v_mov_b32_e32 v1, s5
543; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
544; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
545; GFX7-NEXT:    s_endpgm
546;
547; GFX8-LABEL: idot4_multiuse_mul1:
548; GFX8:       ; %bb.0: ; %entry
549; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
550; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
551; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
552; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
553; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
554; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
555; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
556; GFX8-NEXT:    s_sext_i32_i8 s4, s2
557; GFX8-NEXT:    s_sext_i32_i8 s5, s3
558; GFX8-NEXT:    s_bfe_i32 s7, s3, 0x80008
559; GFX8-NEXT:    v_mov_b32_e32 v0, s5
560; GFX8-NEXT:    v_mov_b32_e32 v1, s10
561; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x80008
562; GFX8-NEXT:    v_mad_i32_i24 v1, s4, v0, v1
563; GFX8-NEXT:    v_mov_b32_e32 v2, s7
564; GFX8-NEXT:    s_bfe_i32 s9, s3, 0x80010
565; GFX8-NEXT:    v_mad_i32_i24 v1, s6, v2, v1
566; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x80010
567; GFX8-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
568; GFX8-NEXT:    v_mov_b32_e32 v1, s9
569; GFX8-NEXT:    s_ashr_i32 s3, s3, 24
570; GFX8-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
571; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
572; GFX8-NEXT:    v_mov_b32_e32 v1, s3
573; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
574; GFX8-NEXT:    v_mov_b32_e32 v0, s0
575; GFX8-NEXT:    v_mov_b32_e32 v1, s1
576; GFX8-NEXT:    flat_store_dword v[0:1], v2
577; GFX8-NEXT:    s_endpgm
578;
579; GFX9-NODL-LABEL: idot4_multiuse_mul1:
580; GFX9-NODL:       ; %bb.0: ; %entry
581; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
582; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
583; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
584; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
585; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
586; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
587; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
588; GFX9-NODL-NEXT:    s_sext_i32_i8 s4, s2
589; GFX9-NODL-NEXT:    s_sext_i32_i8 s5, s3
590; GFX9-NODL-NEXT:    s_bfe_i32 s7, s3, 0x80008
591; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
592; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s10
593; GFX9-NODL-NEXT:    s_bfe_i32 s6, s2, 0x80008
594; GFX9-NODL-NEXT:    v_mad_i32_i24 v1, s4, v0, v1
595; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s7
596; GFX9-NODL-NEXT:    s_bfe_i32 s9, s3, 0x80010
597; GFX9-NODL-NEXT:    v_mad_i32_i24 v1, s6, v2, v1
598; GFX9-NODL-NEXT:    s_bfe_i32 s8, s2, 0x80010
599; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
600; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s9
601; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 24
602; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
603; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
604; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
605; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
606; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
607; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
608; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
609; GFX9-NODL-NEXT:    s_endpgm
610;
611; GFX9-DL-LABEL: idot4_multiuse_mul1:
612; GFX9-DL:       ; %bb.0: ; %entry
613; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
614; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
615; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
616; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
617; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
618; GFX9-DL-NEXT:    s_load_dword s10, s[0:1], 0x0
619; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
620; GFX9-DL-NEXT:    s_sext_i32_i8 s4, s2
621; GFX9-DL-NEXT:    s_sext_i32_i8 s5, s3
622; GFX9-DL-NEXT:    s_bfe_i32 s7, s3, 0x80008
623; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
624; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s10
625; GFX9-DL-NEXT:    s_bfe_i32 s6, s2, 0x80008
626; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s4, v0, v1
627; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s7
628; GFX9-DL-NEXT:    s_bfe_i32 s9, s3, 0x80010
629; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s6, v2, v1
630; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x80010
631; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
632; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s9
633; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 24
634; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
635; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 24
636; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
637; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
638; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
639; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
640; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
641; GFX9-DL-NEXT:    s_endpgm
642;
643; GFX10-DL-LABEL: idot4_multiuse_mul1:
644; GFX10-DL:       ; %bb.0: ; %entry
645; GFX10-DL-NEXT:    s_clause 0x1
646; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
647; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
648; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
649; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
650; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
651; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
652; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
653; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
654; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
655; GFX10-DL-NEXT:    s_sext_i32_i8 s2, s0
656; GFX10-DL-NEXT:    s_sext_i32_i8 s3, s1
657; GFX10-DL-NEXT:    s_bfe_i32 s6, s0, 0x80008
658; GFX10-DL-NEXT:    s_bfe_i32 s7, s1, 0x80008
659; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s2, s3, v0
660; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s7, v0
661; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s2, s3, v0
662; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x80010
663; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x80010
664; GFX10-DL-NEXT:    s_ashr_i32 s0, s0, 24
665; GFX10-DL-NEXT:    s_ashr_i32 s1, s1, 24
666; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s2, s3, v0
667; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s0, s1, v0
668; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
669; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
670; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
671; GFX10-DL-NEXT:    s_endpgm
672                                               <4 x i8> addrspace(1)* %src2,
673                                               i32 addrspace(1)* nocapture %dst) {
674entry:
675  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
676  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
677
678  %v1e0 = extractelement <4 x i8> %vec1, i64 0
679  %cv1e0 = sext i8 %v1e0 to i32
680  %v2e0 = extractelement <4 x i8> %vec2, i64 0
681  %cv2e0 = sext i8 %v2e0 to i32
682  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
683
684  %v1e1 = extractelement <4 x i8> %vec1, i64 1
685  %cv1e1 = sext i8 %v1e1 to i32
686  %v2e1 = extractelement <4 x i8> %vec2, i64 1
687  %cv2e1 = sext i8 %v2e1 to i32
688  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
689
690  %v1e2 = extractelement <4 x i8> %vec1, i64 2
691  %cv1e2 = sext i8 %v1e2 to i32
692  %v2e2 = extractelement <4 x i8> %vec2, i64 2
693  %cv2e2 = sext i8 %v2e2 to i32
694  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
695
696  %v1e3 = extractelement <4 x i8> %vec1, i64 3
697  %cv1e3 = sext i8 %v1e3 to i32
698  %v2e3 = extractelement <4 x i8> %vec2, i64 3
699  %cv2e3 = sext i8 %v2e3 to i32
700  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
701
702  %acc = load i32, i32 addrspace(1)* %dst, align 4
703  %add = add i32 %mul1, %acc
704  %add1 = add i32 %mul2, %add
705  %add2 = add i32 %add1, %mul1
706  %add3 = add i32 %add2, %mul3
707  %add4 = add i32 %add3, %mul4
708
709  store i32 %add4, i32 addrspace(1)* %dst, align 4
710  ret void
711}
712
713; TODO: Support this pattern.
714define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
715; GFX7-LABEL: idot4_acc32_vecMul:
716; GFX7:       ; %bb.0: ; %entry
717; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
718; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
719; GFX7-NEXT:    s_mov_b32 s3, 0xf000
720; GFX7-NEXT:    s_mov_b32 s2, -1
721; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
722; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
723; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
724; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
725; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
726; GFX7-NEXT:    s_ashr_i32 s6, s4, 24
727; GFX7-NEXT:    s_ashr_i32 s9, s5, 24
728; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80010
729; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80008
730; GFX7-NEXT:    s_sext_i32_i8 s5, s5
731; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x80010
732; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80008
733; GFX7-NEXT:    s_sext_i32_i8 s4, s4
734; GFX7-NEXT:    v_mov_b32_e32 v0, s5
735; GFX7-NEXT:    v_mov_b32_e32 v1, s12
736; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
737; GFX7-NEXT:    v_mov_b32_e32 v1, s11
738; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
739; GFX7-NEXT:    v_mov_b32_e32 v1, s10
740; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v1, v0
741; GFX7-NEXT:    v_mov_b32_e32 v1, s9
742; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
743; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
744; GFX7-NEXT:    s_endpgm
745;
746; GFX8-LABEL: idot4_acc32_vecMul:
747; GFX8:       ; %bb.0: ; %entry
748; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
749; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
750; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
751; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
752; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
753; GFX8-NEXT:    s_load_dword s8, s[0:1], 0x0
754; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
755; GFX8-NEXT:    v_lshrrev_b16_e64 v0, 8, s2
756; GFX8-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
757; GFX8-NEXT:    s_ashr_i32 s6, s3, 24
758; GFX8-NEXT:    s_bfe_i32 s7, s3, 0x80010
759; GFX8-NEXT:    s_sext_i32_i8 s3, s3
760; GFX8-NEXT:    s_ashr_i32 s4, s2, 24
761; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x80010
762; GFX8-NEXT:    s_sext_i32_i8 s2, s2
763; GFX8-NEXT:    v_mov_b32_e32 v2, s3
764; GFX8-NEXT:    v_mov_b32_e32 v3, s8
765; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 8
766; GFX8-NEXT:    v_bfe_i32 v1, v1, 0, 8
767; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v2, v3
768; GFX8-NEXT:    v_mad_i32_i24 v0, v0, v1, v2
769; GFX8-NEXT:    v_mov_b32_e32 v1, s7
770; GFX8-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
771; GFX8-NEXT:    v_mov_b32_e32 v1, s6
772; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v1, v0
773; GFX8-NEXT:    v_mov_b32_e32 v0, s0
774; GFX8-NEXT:    v_mov_b32_e32 v1, s1
775; GFX8-NEXT:    flat_store_dword v[0:1], v2
776; GFX8-NEXT:    s_endpgm
777;
778; GFX9-NODL-LABEL: idot4_acc32_vecMul:
779; GFX9-NODL:       ; %bb.0: ; %entry
780; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
781; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
782; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
783; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
784; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
785; GFX9-NODL-NEXT:    s_load_dword s8, s[0:1], 0x0
786; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
787; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v0, 8, s2
788; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
789; GFX9-NODL-NEXT:    s_ashr_i32 s6, s3, 24
790; GFX9-NODL-NEXT:    s_bfe_i32 s7, s3, 0x80010
791; GFX9-NODL-NEXT:    s_sext_i32_i8 s3, s3
792; GFX9-NODL-NEXT:    s_ashr_i32 s4, s2, 24
793; GFX9-NODL-NEXT:    s_bfe_i32 s5, s2, 0x80010
794; GFX9-NODL-NEXT:    s_sext_i32_i8 s2, s2
795; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s3
796; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s8
797; GFX9-NODL-NEXT:    v_bfe_i32 v0, v0, 0, 8
798; GFX9-NODL-NEXT:    v_bfe_i32 v1, v1, 0, 8
799; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v2, v3
800; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, v0, v1, v2
801; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s7
802; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
803; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
804; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s4, v1, v0
805; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
806; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
807; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
808; GFX9-NODL-NEXT:    s_endpgm
809;
810; GFX9-DL-LABEL: idot4_acc32_vecMul:
811; GFX9-DL:       ; %bb.0: ; %entry
812; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
813; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
814; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
815; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
816; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
817; GFX9-DL-NEXT:    s_load_dword s8, s[0:1], 0x0
818; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
819; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v0, 8, s2
820; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
821; GFX9-DL-NEXT:    s_ashr_i32 s6, s3, 24
822; GFX9-DL-NEXT:    s_bfe_i32 s7, s3, 0x80010
823; GFX9-DL-NEXT:    s_sext_i32_i8 s3, s3
824; GFX9-DL-NEXT:    s_ashr_i32 s4, s2, 24
825; GFX9-DL-NEXT:    s_bfe_i32 s5, s2, 0x80010
826; GFX9-DL-NEXT:    s_sext_i32_i8 s2, s2
827; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
828; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s8
829; GFX9-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
830; GFX9-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
831; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v2, v3
832; GFX9-DL-NEXT:    v_mad_i32_i24 v0, v0, v1, v2
833; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s7
834; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
835; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
836; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s4, v1, v0
837; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
838; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
839; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
840; GFX9-DL-NEXT:    s_endpgm
841;
842; GFX10-DL-LABEL: idot4_acc32_vecMul:
843; GFX10-DL:       ; %bb.0: ; %entry
844; GFX10-DL-NEXT:    s_clause 0x1
845; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
846; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
847; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
848; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
849; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
850; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
851; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
852; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
853; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v0, 8, s2
854; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
855; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
856; GFX10-DL-NEXT:    s_sext_i32_i8 s4, s2
857; GFX10-DL-NEXT:    s_sext_i32_i8 s5, s3
858; GFX10-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
859; GFX10-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
860; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s4, s5, v2
861; GFX10-DL-NEXT:    s_bfe_i32 s4, s2, 0x80010
862; GFX10-DL-NEXT:    s_bfe_i32 s5, s3, 0x80010
863; GFX10-DL-NEXT:    s_ashr_i32 s2, s2, 24
864; GFX10-DL-NEXT:    s_ashr_i32 s3, s3, 24
865; GFX10-DL-NEXT:    v_mad_i32_i24 v0, v0, v1, v2
866; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s4, s5, v0
867; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s3, v0
868; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
869; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
870; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
871; GFX10-DL-NEXT:    s_endpgm
872                                              <4 x i8> addrspace(1)* %src2,
873                                              i32 addrspace(1)* nocapture %dst) {
874entry:
875  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
876  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
877
878  %cvec1 = sext <4 x i8> %vec1 to <4 x i32>
879  %cvec2 = sext <4 x i8> %vec2 to <4 x i32>
880
881  %mul = mul <4 x i32> %cvec1, %cvec2
882  %mul0 = extractelement <4 x i32> %mul, i64 0
883  %mul1 = extractelement <4 x i32> %mul, i64 1
884  %mul2 = extractelement <4 x i32> %mul, i64 2
885  %mul3 = extractelement <4 x i32> %mul, i64 3
886
887  %acc = load i32, i32 addrspace(1)* %dst, align 4
888  %add1 = add i32 %mul0, %acc
889  %add2 = add i32 %add1, %mul1
890  %add3 = add i32 %add2, %mul2
891  %add4 = add i32 %add3, %mul3
892
893  store i32 %add4, i32 addrspace(1)* %dst, align 4
894  ret void
895}
896
897define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
898; GFX7-LABEL: idot4_acc16_vecMul:
899; GFX7:       ; %bb.0: ; %entry
900; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
901; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
902; GFX7-NEXT:    s_mov_b32 s3, 0xf000
903; GFX7-NEXT:    s_mov_b32 s2, -1
904; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
905; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
906; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
907; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
908; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
909; GFX7-NEXT:    s_ashr_i32 s6, s4, 24
910; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80010
911; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80008
912; GFX7-NEXT:    s_ashr_i32 s9, s5, 24
913; GFX7-NEXT:    s_sext_i32_i8 s5, s5
914; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x80010
915; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80008
916; GFX7-NEXT:    s_sext_i32_i8 s4, s4
917; GFX7-NEXT:    v_mov_b32_e32 v1, s5
918; GFX7-NEXT:    v_mov_b32_e32 v2, s11
919; GFX7-NEXT:    v_mov_b32_e32 v3, s10
920; GFX7-NEXT:    s_waitcnt vmcnt(0)
921; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
922; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v2, v0
923; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v3, v0
924; GFX7-NEXT:    v_mov_b32_e32 v1, s9
925; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
926; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
927; GFX7-NEXT:    s_endpgm
928;
929; GFX8-LABEL: idot4_acc16_vecMul:
930; GFX8:       ; %bb.0: ; %entry
931; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
932; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
933; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
934; GFX8-NEXT:    v_mov_b32_e32 v0, s0
935; GFX8-NEXT:    v_mov_b32_e32 v1, s1
936; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
937; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
938; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
939; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
940; GFX8-NEXT:    v_lshrrev_b16_e64 v3, 8, s0
941; GFX8-NEXT:    v_lshrrev_b16_e64 v4, 8, s1
942; GFX8-NEXT:    s_bfe_i32 s5, s1, 0x80010
943; GFX8-NEXT:    s_ashr_i32 s4, s1, 24
944; GFX8-NEXT:    s_sext_i32_i8 s1, s1
945; GFX8-NEXT:    s_ashr_i32 s2, s0, 24
946; GFX8-NEXT:    s_bfe_i32 s3, s0, 0x80010
947; GFX8-NEXT:    s_sext_i32_i8 s0, s0
948; GFX8-NEXT:    v_mov_b32_e32 v5, s1
949; GFX8-NEXT:    v_bfe_i32 v3, v3, 0, 8
950; GFX8-NEXT:    v_bfe_i32 v4, v4, 0, 8
951; GFX8-NEXT:    v_mov_b32_e32 v6, s5
952; GFX8-NEXT:    s_waitcnt vmcnt(0)
953; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v5, v2
954; GFX8-NEXT:    v_mad_i32_i24 v2, v3, v4, v2
955; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v6, v2
956; GFX8-NEXT:    v_mov_b32_e32 v3, s4
957; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
958; GFX8-NEXT:    flat_store_short v[0:1], v2
959; GFX8-NEXT:    s_endpgm
960;
961; GFX9-NODL-LABEL: idot4_acc16_vecMul:
962; GFX9-NODL:       ; %bb.0: ; %entry
963; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
964; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
965; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, 0xffff
966; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
967; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
968; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
969; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
970; GFX9-NODL-NEXT:    s_lshr_b32 s4, s2, 16
971; GFX9-NODL-NEXT:    s_lshr_b32 s5, s3, 16
972; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v3, 8, s5
973; GFX9-NODL-NEXT:    s_bfe_i32 s5, s5, 0x80000
974; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v2, 8, s4
975; GFX9-NODL-NEXT:    v_and_b32_e32 v5, s5, v4
976; GFX9-NODL-NEXT:    s_bfe_i32 s4, s4, 0x80000
977; GFX9-NODL-NEXT:    v_lshl_or_b32 v3, v3, 16, v5
978; GFX9-NODL-NEXT:    v_and_b32_e32 v5, s4, v4
979; GFX9-NODL-NEXT:    v_lshl_or_b32 v2, v2, 16, v5
980; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v1, 8, s3
981; GFX9-NODL-NEXT:    s_bfe_i32 s3, s3, 0x80000
982; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v0, 8, s2
983; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v2, v2, v3
984; GFX9-NODL-NEXT:    v_and_b32_e32 v3, s3, v4
985; GFX9-NODL-NEXT:    s_bfe_i32 s2, s2, 0x80000
986; GFX9-NODL-NEXT:    v_lshl_or_b32 v1, v1, 16, v3
987; GFX9-NODL-NEXT:    v_and_b32_e32 v3, s2, v4
988; GFX9-NODL-NEXT:    v_lshl_or_b32 v0, v0, 16, v3
989; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v3, v0, v1
990; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
991; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
992; GFX9-NODL-NEXT:    global_load_ushort v4, v[0:1], off
993; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
994; GFX9-NODL-NEXT:    v_add_u32_e32 v4, v3, v4
995; GFX9-NODL-NEXT:    v_add_u32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
996; GFX9-NODL-NEXT:    v_add_u32_e32 v3, v3, v2
997; GFX9-NODL-NEXT:    v_add_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
998; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
999; GFX9-NODL-NEXT:    s_endpgm
1000;
1001; GFX9-DL-LABEL: idot4_acc16_vecMul:
1002; GFX9-DL:       ; %bb.0: ; %entry
1003; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1004; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1005; GFX9-DL-NEXT:    v_mov_b32_e32 v4, 0xffff
1006; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1007; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1008; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1009; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1010; GFX9-DL-NEXT:    s_lshr_b32 s4, s2, 16
1011; GFX9-DL-NEXT:    s_lshr_b32 s5, s3, 16
1012; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v3, 8, s5
1013; GFX9-DL-NEXT:    s_bfe_i32 s5, s5, 0x80000
1014; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v2, 8, s4
1015; GFX9-DL-NEXT:    v_and_b32_e32 v5, s5, v4
1016; GFX9-DL-NEXT:    s_bfe_i32 s4, s4, 0x80000
1017; GFX9-DL-NEXT:    v_lshl_or_b32 v3, v3, 16, v5
1018; GFX9-DL-NEXT:    v_and_b32_e32 v5, s4, v4
1019; GFX9-DL-NEXT:    v_lshl_or_b32 v2, v2, 16, v5
1020; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v1, 8, s3
1021; GFX9-DL-NEXT:    s_bfe_i32 s3, s3, 0x80000
1022; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v0, 8, s2
1023; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v3
1024; GFX9-DL-NEXT:    v_and_b32_e32 v3, s3, v4
1025; GFX9-DL-NEXT:    s_bfe_i32 s2, s2, 0x80000
1026; GFX9-DL-NEXT:    v_lshl_or_b32 v1, v1, 16, v3
1027; GFX9-DL-NEXT:    v_and_b32_e32 v3, s2, v4
1028; GFX9-DL-NEXT:    v_lshl_or_b32 v0, v0, 16, v3
1029; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, v0, v1
1030; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1031; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1032; GFX9-DL-NEXT:    global_load_ushort v4, v[0:1], off
1033; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
1034; GFX9-DL-NEXT:    v_add_u32_e32 v4, v3, v4
1035; GFX9-DL-NEXT:    v_add_u32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1036; GFX9-DL-NEXT:    v_add_u32_e32 v3, v3, v2
1037; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1038; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
1039; GFX9-DL-NEXT:    s_endpgm
1040;
1041; GFX10-DL-LABEL: idot4_acc16_vecMul:
1042; GFX10-DL:       ; %bb.0: ; %entry
1043; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
1044; GFX10-DL-NEXT:    v_mov_b32_e32 v3, 0xffff
1045; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1046; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1047; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
1048; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s3
1049; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1050; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
1051; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1052; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
1053; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
1054; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1055; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 16
1056; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 8, s0
1057; GFX10-DL-NEXT:    s_bfe_i32 s0, s0, 0x80000
1058; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x80000
1059; GFX10-DL-NEXT:    v_and_b32_e32 v7, s0, v3
1060; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 8, s1
1061; GFX10-DL-NEXT:    v_and_b32_e32 v6, s3, v3
1062; GFX10-DL-NEXT:    s_lshr_b32 s0, s1, 16
1063; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v8, 8, s2
1064; GFX10-DL-NEXT:    v_lshl_or_b32 v4, v4, 16, v7
1065; GFX10-DL-NEXT:    s_bfe_i32 s1, s2, 0x80000
1066; GFX10-DL-NEXT:    v_lshl_or_b32 v5, v5, 16, v6
1067; GFX10-DL-NEXT:    s_bfe_i32 s2, s0, 0x80000
1068; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v6, 8, s0
1069; GFX10-DL-NEXT:    v_and_b32_e32 v7, s2, v3
1070; GFX10-DL-NEXT:    v_and_b32_e32 v3, s1, v3
1071; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v5
1072; GFX10-DL-NEXT:    v_lshl_or_b32 v5, v6, 16, v7
1073; GFX10-DL-NEXT:    v_lshl_or_b32 v3, v8, 16, v3
1074; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v5
1075; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
1076; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v4, v2
1077; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1078; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v3
1079; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1080; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
1081; GFX10-DL-NEXT:    s_endpgm
1082                                              <4 x i8> addrspace(1)* %src2,
1083                                              i16 addrspace(1)* nocapture %dst) {
1084entry:
1085  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
1086  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
1087
1088  %cvec1 = sext <4 x i8> %vec1 to <4 x i16>
1089  %cvec2 = sext <4 x i8> %vec2 to <4 x i16>
1090
1091  %mul = mul <4 x i16> %cvec1, %cvec2
1092  %mul0 = extractelement <4 x i16> %mul, i64 0
1093  %mul1 = extractelement <4 x i16> %mul, i64 1
1094  %mul2 = extractelement <4 x i16> %mul, i64 2
1095  %mul3 = extractelement <4 x i16> %mul, i64 3
1096
1097  %acc = load i16, i16 addrspace(1)* %dst, align 4
1098  %add1 = add i16 %mul0, %acc
1099  %add2 = add i16 %add1, %mul1
1100  %add3 = add i16 %add2, %mul2
1101  %add4 = add i16 %add3, %mul3
1102
1103  store i16 %add4, i16 addrspace(1)* %dst, align 4
1104  ret void
1105}
1106