1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-NODL %s
5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
8
9define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
10; GFX7-LABEL: idot4_acc32:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
13; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
14; GFX7-NEXT:    s_mov_b32 s3, 0xf000
15; GFX7-NEXT:    s_mov_b32 s2, -1
16; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
17; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
18; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
19; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
20; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX7-NEXT:    s_sext_i32_i8 s6, s4
22; GFX7-NEXT:    s_sext_i32_i8 s7, s5
23; GFX7-NEXT:    s_bfe_i32 s9, s5, 0x80008
24; GFX7-NEXT:    v_mov_b32_e32 v0, s7
25; GFX7-NEXT:    v_mov_b32_e32 v1, s12
26; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80010
27; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v0, v1
28; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80008
29; GFX7-NEXT:    v_mov_b32_e32 v1, s9
30; GFX7-NEXT:    s_bfe_i32 s10, s4, 0x80010
31; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
32; GFX7-NEXT:    v_mov_b32_e32 v1, s11
33; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
34; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
35; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
36; GFX7-NEXT:    v_mov_b32_e32 v1, s5
37; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
38; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
39; GFX7-NEXT:    s_endpgm
40;
41; GFX8-LABEL: idot4_acc32:
42; GFX8:       ; %bb.0: ; %entry
43; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
44; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
45; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
46; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
47; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
48; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
49; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
50; GFX8-NEXT:    s_sext_i32_i8 s4, s2
51; GFX8-NEXT:    s_sext_i32_i8 s5, s3
52; GFX8-NEXT:    s_bfe_i32 s7, s3, 0x80008
53; GFX8-NEXT:    v_mov_b32_e32 v0, s5
54; GFX8-NEXT:    v_mov_b32_e32 v1, s10
55; GFX8-NEXT:    s_bfe_i32 s9, s3, 0x80010
56; GFX8-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
57; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x80008
58; GFX8-NEXT:    v_mov_b32_e32 v1, s7
59; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x80010
60; GFX8-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
61; GFX8-NEXT:    v_mov_b32_e32 v1, s9
62; GFX8-NEXT:    s_ashr_i32 s3, s3, 24
63; GFX8-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
64; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
65; GFX8-NEXT:    v_mov_b32_e32 v1, s3
66; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
67; GFX8-NEXT:    v_mov_b32_e32 v0, s0
68; GFX8-NEXT:    v_mov_b32_e32 v1, s1
69; GFX8-NEXT:    flat_store_dword v[0:1], v2
70; GFX8-NEXT:    s_endpgm
71;
72; GFX9-NODL-LABEL: idot4_acc32:
73; GFX9-NODL:       ; %bb.0: ; %entry
74; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
75; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
76; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
77; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
78; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
79; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
80; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
81; GFX9-NODL-NEXT:    s_sext_i32_i8 s4, s2
82; GFX9-NODL-NEXT:    s_sext_i32_i8 s5, s3
83; GFX9-NODL-NEXT:    s_bfe_i32 s7, s3, 0x80008
84; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
85; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s10
86; GFX9-NODL-NEXT:    s_bfe_i32 s9, s3, 0x80010
87; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
88; GFX9-NODL-NEXT:    s_bfe_i32 s6, s2, 0x80008
89; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s7
90; GFX9-NODL-NEXT:    s_bfe_i32 s8, s2, 0x80010
91; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
92; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s9
93; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 24
94; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
95; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
96; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
97; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
98; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
99; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
100; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
101; GFX9-NODL-NEXT:    s_endpgm
102;
103; GFX9-DL-LABEL: idot4_acc32:
104; GFX9-DL:       ; %bb.0: ; %entry
105; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
106; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
107; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
108; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
109; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
110; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
111; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
112; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
113; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
114; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
115; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
116; GFX9-DL-NEXT:    v_dot4_i32_i8 v2, s4, v2, v3
117; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
118; GFX9-DL-NEXT:    s_endpgm
119;
120; GFX10-DL-LABEL: idot4_acc32:
121; GFX10-DL:       ; %bb.0: ; %entry
122; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
123; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
124; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
125; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
126; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
127; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
128; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
129; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
130; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
131; GFX10-DL-NEXT:    v_dot4_i32_i8 v2, s3, s4, v0
132; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
133; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
134; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
135; GFX10-DL-NEXT:    s_endpgm
136                                       <4 x i8> addrspace(1)* %src2,
137                                       i32 addrspace(1)* nocapture %dst) {
138entry:
139  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
140  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
141
142  %v1e0 = extractelement <4 x i8> %vec1, i64 0
143  %cv1e0 = sext i8 %v1e0 to i32
144  %v2e0 = extractelement <4 x i8> %vec2, i64 0
145  %cv2e0 = sext i8 %v2e0 to i32
146  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
147
148  %v1e1 = extractelement <4 x i8> %vec1, i64 1
149  %cv1e1 = sext i8 %v1e1 to i32
150  %v2e1 = extractelement <4 x i8> %vec2, i64 1
151  %cv2e1 = sext i8 %v2e1 to i32
152  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
153
154  %v1e2 = extractelement <4 x i8> %vec1, i64 2
155  %cv1e2 = sext i8 %v1e2 to i32
156  %v2e2 = extractelement <4 x i8> %vec2, i64 2
157  %cv2e2 = sext i8 %v2e2 to i32
158  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
159
160  %v1e3 = extractelement <4 x i8> %vec1, i64 3
161  %cv1e3 = sext i8 %v1e3 to i32
162  %v2e3 = extractelement <4 x i8> %vec2, i64 3
163  %cv2e3 = sext i8 %v2e3 to i32
164  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
165
166  %acc = load i32, i32 addrspace(1)* %dst, align 4
167  %add1 = add i32 %mul1, %acc
168  %add2 = add i32 %add1, %mul2
169  %add3 = add i32 %add2, %mul3
170  %add4 = add i32 %add3, %mul4
171  store i32 %add4, i32 addrspace(1)* %dst, align 4
172  ret void
173}
174
175; TODO: Currently, vector elements{0 and 3} get zero_extended from i16 to i32 which should
176; be sign_extended directly to i32; prevents the pattern recognizer to recognize this pattern.
177define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
178; GFX7-LABEL: idot4_acc16:
179; GFX7:       ; %bb.0: ; %entry
180; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
181; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
182; GFX7-NEXT:    s_mov_b32 s3, 0xf000
183; GFX7-NEXT:    s_mov_b32 s2, -1
184; GFX7-NEXT:    s_mov_b32 s8, 0xffff
185; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
186; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
187; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
188; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
189; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
190; GFX7-NEXT:    s_sext_i32_i8 s6, s4
191; GFX7-NEXT:    s_sext_i32_i8 s7, s5
192; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80008
193; GFX7-NEXT:    s_and_b32 s7, s7, s8
194; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
195; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
196; GFX7-NEXT:    s_and_b32 s10, s10, s8
197; GFX7-NEXT:    s_and_b32 s6, s6, s8
198; GFX7-NEXT:    v_mov_b32_e32 v1, s7
199; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x80010
200; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
201; GFX7-NEXT:    s_and_b32 s12, s12, s8
202; GFX7-NEXT:    s_and_b32 s9, s9, s8
203; GFX7-NEXT:    v_mov_b32_e32 v2, s10
204; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
205; GFX7-NEXT:    s_and_b32 s11, s11, s8
206; GFX7-NEXT:    s_and_b32 s5, s5, s8
207; GFX7-NEXT:    v_mov_b32_e32 v3, s12
208; GFX7-NEXT:    s_and_b32 s4, s4, s8
209; GFX7-NEXT:    s_waitcnt vmcnt(0)
210; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
211; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
212; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
213; GFX7-NEXT:    v_mov_b32_e32 v1, s5
214; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
215; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
216; GFX7-NEXT:    s_endpgm
217;
218; GFX8-LABEL: idot4_acc16:
219; GFX8:       ; %bb.0: ; %entry
220; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
221; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
222; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
223; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
224; GFX8-NEXT:    v_mov_b32_e32 v0, s0
225; GFX8-NEXT:    v_mov_b32_e32 v1, s1
226; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
227; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
228; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
229; GFX8-NEXT:    s_sext_i32_i8 s1, s2
230; GFX8-NEXT:    s_bfe_i32 s3, s2, 0x80008
231; GFX8-NEXT:    v_mov_b32_e32 v3, s1
232; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x80010
233; GFX8-NEXT:    s_sext_i32_i8 s1, s0
234; GFX8-NEXT:    v_mov_b32_e32 v4, s3
235; GFX8-NEXT:    s_bfe_i32 s4, s0, 0x80008
236; GFX8-NEXT:    s_bfe_i32 s3, s0, 0x80010
237; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
238; GFX8-NEXT:    v_mov_b32_e32 v5, s5
239; GFX8-NEXT:    s_ashr_i32 s0, s0, 24
240; GFX8-NEXT:    s_waitcnt vmcnt(0)
241; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
242; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v4, v2
243; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v5, v2
244; GFX8-NEXT:    v_mov_b32_e32 v3, s2
245; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
246; GFX8-NEXT:    flat_store_short v[0:1], v2
247; GFX8-NEXT:    s_endpgm
248;
249; GFX9-NODL-LABEL: idot4_acc16:
250; GFX9-NODL:       ; %bb.0: ; %entry
251; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
252; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
253; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
254; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
255; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
256; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
257; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
258; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
259; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
260; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s2
261; GFX9-NODL-NEXT:    s_bfe_i32 s3, s2, 0x80008
262; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
263; GFX9-NODL-NEXT:    s_bfe_i32 s5, s2, 0x80010
264; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s0
265; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s3
266; GFX9-NODL-NEXT:    s_bfe_i32 s4, s0, 0x80008
267; GFX9-NODL-NEXT:    s_bfe_i32 s3, s0, 0x80010
268; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
269; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s5
270; GFX9-NODL-NEXT:    s_ashr_i32 s0, s0, 24
271; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
272; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
273; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s4, v4, v2
274; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v5, v2
275; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
276; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
277; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
278; GFX9-NODL-NEXT:    s_endpgm
279;
280; GFX9-DL-LABEL: idot4_acc16:
281; GFX9-DL:       ; %bb.0: ; %entry
282; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
283; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
284; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
285; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
286; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
287; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
288; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
289; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
290; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
291; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
292; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
293; GFX9-DL-NEXT:    v_dot4_i32_i8 v2, s0, v3, v2
294; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
295; GFX9-DL-NEXT:    s_endpgm
296;
297; GFX10-DL-LABEL: idot4_acc16:
298; GFX10-DL:       ; %bb.0: ; %entry
299; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
300; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
301; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
302; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
303; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s3
304; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
305; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
306; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
307; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
308; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
309; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
310; GFX10-DL-NEXT:    v_dot4_i32_i8 v2, s0, s1, v2
311; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
312; GFX10-DL-NEXT:    s_endpgm
313                                       <4 x i8> addrspace(1)* %src2,
314                                       i16 addrspace(1)* nocapture %dst) {
315entry:
316  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
317  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
318
319  %v1e0 = extractelement <4 x i8> %vec1, i64 0
320  %cv1e0 = sext i8 %v1e0 to i16
321  %v2e0 = extractelement <4 x i8> %vec2, i64 0
322  %cv2e0 = sext i8 %v2e0 to i16
323  %mul1 = mul nsw i16 %cv1e0, %cv2e0
324
325  %v1e1 = extractelement <4 x i8> %vec1, i64 1
326  %cv1e1 = sext i8 %v1e1 to i16
327  %v2e1 = extractelement <4 x i8> %vec2, i64 1
328  %cv2e1 = sext i8 %v2e1 to i16
329  %mul2 = mul nsw i16 %cv1e1, %cv2e1
330
331  %v1e2 = extractelement <4 x i8> %vec1, i64 2
332  %cv1e2 = sext i8 %v1e2 to i16
333  %v2e2 = extractelement <4 x i8> %vec2, i64 2
334  %cv2e2 = sext i8 %v2e2 to i16
335  %mul3 = mul nsw i16 %cv1e2, %cv2e2
336
337  %v1e3 = extractelement <4 x i8> %vec1, i64 3
338  %cv1e3 = sext i8 %v1e3 to i16
339  %v2e3 = extractelement <4 x i8> %vec2, i64 3
340  %cv2e3 = sext i8 %v2e3 to i16
341  %mul4 = mul nsw i16 %cv1e3, %cv2e3
342
343  %acc = load i16, i16 addrspace(1)* %dst, align 2
344  %add1 = add i16 %mul1, %acc
345  %add2 = add i16 %add1, %mul2
346  %add3 = add i16 %add2, %mul3
347  %add4 = add i16 %add3, %mul4
348  store i16 %add4, i16 addrspace(1)* %dst, align 2
349  ret void
350}
351
352define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
353; GFX7-LABEL: idot4_acc8:
354; GFX7:       ; %bb.0: ; %entry
355; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
356; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
357; GFX7-NEXT:    s_mov_b32 s3, 0xf000
358; GFX7-NEXT:    s_mov_b32 s2, -1
359; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
360; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
361; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
362; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
363; GFX7-NEXT:    s_movk_i32 s5, 0xff
364; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
365; GFX7-NEXT:    s_and_b32 s7, s6, s5
366; GFX7-NEXT:    s_bfe_u32 s8, s6, 0x80008
367; GFX7-NEXT:    s_and_b32 s5, s4, s5
368; GFX7-NEXT:    v_mov_b32_e32 v1, s7
369; GFX7-NEXT:    s_bfe_u32 s10, s6, 0x80010
370; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
371; GFX7-NEXT:    v_mov_b32_e32 v2, s8
372; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
373; GFX7-NEXT:    s_lshr_b32 s6, s6, 24
374; GFX7-NEXT:    v_mov_b32_e32 v3, s10
375; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
376; GFX7-NEXT:    s_waitcnt vmcnt(0)
377; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
378; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
379; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
380; GFX7-NEXT:    v_mov_b32_e32 v1, s6
381; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
382; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
383; GFX7-NEXT:    s_endpgm
384;
385; GFX8-LABEL: idot4_acc8:
386; GFX8:       ; %bb.0: ; %entry
387; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
388; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
389; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
390; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
391; GFX8-NEXT:    v_mov_b32_e32 v0, s0
392; GFX8-NEXT:    v_mov_b32_e32 v1, s1
393; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
394; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
395; GFX8-NEXT:    s_movk_i32 s0, 0xff
396; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
397; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
398; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
399; GFX8-NEXT:    s_and_b32 s3, s1, s0
400; GFX8-NEXT:    s_and_b32 s0, s2, s0
401; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x80008
402; GFX8-NEXT:    v_mov_b32_e32 v3, s3
403; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
404; GFX8-NEXT:    v_mov_b32_e32 v4, s4
405; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
406; GFX8-NEXT:    v_mov_b32_e32 v5, s6
407; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
408; GFX8-NEXT:    s_waitcnt vmcnt(0)
409; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
410; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
411; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
412; GFX8-NEXT:    v_mov_b32_e32 v3, s1
413; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
414; GFX8-NEXT:    flat_store_byte v[0:1], v2
415; GFX8-NEXT:    s_endpgm
416;
417; GFX9-NODL-LABEL: idot4_acc8:
418; GFX9-NODL:       ; %bb.0: ; %entry
419; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
420; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
421; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
422; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
423; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
424; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
425; GFX9-NODL-NEXT:    global_load_ubyte v2, v[0:1], off
426; GFX9-NODL-NEXT:    s_load_dword s1, s[6:7], 0x0
427; GFX9-NODL-NEXT:    s_movk_i32 s0, 0xff
428; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
429; GFX9-NODL-NEXT:    s_bfe_u32 s5, s2, 0x80008
430; GFX9-NODL-NEXT:    s_bfe_u32 s7, s2, 0x80010
431; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s0
432; GFX9-NODL-NEXT:    s_and_b32 s0, s2, s0
433; GFX9-NODL-NEXT:    s_bfe_u32 s4, s1, 0x80008
434; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
435; GFX9-NODL-NEXT:    s_bfe_u32 s6, s1, 0x80010
436; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s4
437; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 24
438; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s6
439; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
440; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
441; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
442; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
443; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
444; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
445; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
446; GFX9-NODL-NEXT:    global_store_byte v[0:1], v2, off
447; GFX9-NODL-NEXT:    s_endpgm
448;
449; GFX9-DL-LABEL: idot4_acc8:
450; GFX9-DL:       ; %bb.0: ; %entry
451; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
452; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
453; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
454; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
455; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
456; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
457; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
458; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
459; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
460; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
461; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
462; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s0, v3, v2
463; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
464; GFX9-DL-NEXT:    s_endpgm
465;
466; GFX10-DL-LABEL: idot4_acc8:
467; GFX10-DL:       ; %bb.0: ; %entry
468; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
469; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
470; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
471; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
472; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s3
473; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
474; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
475; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
476; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
477; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
478; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
479; GFX10-DL-NEXT:    v_dot4_u32_u8 v2, s0, s1, v2
480; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
481; GFX10-DL-NEXT:    s_endpgm
482                                      <4 x i8> addrspace(1)* %src2,
483                                      i8 addrspace(1)* nocapture %dst) {
484entry:
485  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
486  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
487
488  %v1e0 = extractelement <4 x i8> %vec1, i64 0
489  %v2e0 = extractelement <4 x i8> %vec2, i64 0
490  %mul1 = mul i8 %v1e0, %v2e0
491
492  %v1e1 = extractelement <4 x i8> %vec1, i64 1
493  %v2e1 = extractelement <4 x i8> %vec2, i64 1
494  %mul2 = mul i8 %v1e1, %v2e1
495
496  %v1e2 = extractelement <4 x i8> %vec1, i64 2
497  %v2e2 = extractelement <4 x i8> %vec2, i64 2
498  %mul3 = mul i8 %v1e2, %v2e2
499
500  %v1e3 = extractelement <4 x i8> %vec1, i64 3
501  %v2e3 = extractelement <4 x i8> %vec2, i64 3
502  %mul4 = mul i8 %v1e3, %v2e3
503
504  %acc = load i8, i8 addrspace(1)* %dst, align 2
505  %add1 = add i8 %mul1, %acc
506  %add2 = add i8 %add1, %mul2
507  %add3 = add i8 %add2, %mul3
508  %add4 = add nsw i8 %add3, %mul4
509  store i8 %add4, i8 addrspace(1)* %dst, align 2
510  ret void
511}
512
513define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
514; GFX7-LABEL: idot4_multiuse_mul1:
515; GFX7:       ; %bb.0: ; %entry
516; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
517; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
518; GFX7-NEXT:    s_mov_b32 s3, 0xf000
519; GFX7-NEXT:    s_mov_b32 s2, -1
520; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
521; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
522; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
523; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
524; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
525; GFX7-NEXT:    s_sext_i32_i8 s6, s4
526; GFX7-NEXT:    s_sext_i32_i8 s7, s5
527; GFX7-NEXT:    s_bfe_i32 s9, s5, 0x80008
528; GFX7-NEXT:    v_mov_b32_e32 v0, s7
529; GFX7-NEXT:    v_mov_b32_e32 v1, s12
530; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80008
531; GFX7-NEXT:    v_mad_i32_i24 v1, s6, v0, v1
532; GFX7-NEXT:    v_mov_b32_e32 v2, s9
533; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80010
534; GFX7-NEXT:    v_mad_i32_i24 v1, s8, v2, v1
535; GFX7-NEXT:    s_bfe_i32 s10, s4, 0x80010
536; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v0, v1
537; GFX7-NEXT:    v_mov_b32_e32 v1, s11
538; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
539; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
540; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
541; GFX7-NEXT:    v_mov_b32_e32 v1, s5
542; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
543; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
544; GFX7-NEXT:    s_endpgm
545;
546; GFX8-LABEL: idot4_multiuse_mul1:
547; GFX8:       ; %bb.0: ; %entry
548; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
549; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
550; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
551; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
552; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
553; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
554; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
555; GFX8-NEXT:    s_sext_i32_i8 s4, s2
556; GFX8-NEXT:    s_sext_i32_i8 s5, s3
557; GFX8-NEXT:    s_bfe_i32 s7, s3, 0x80008
558; GFX8-NEXT:    v_mov_b32_e32 v0, s5
559; GFX8-NEXT:    v_mov_b32_e32 v1, s10
560; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x80008
561; GFX8-NEXT:    v_mad_i32_i24 v1, s4, v0, v1
562; GFX8-NEXT:    v_mov_b32_e32 v2, s7
563; GFX8-NEXT:    s_bfe_i32 s9, s3, 0x80010
564; GFX8-NEXT:    v_mad_i32_i24 v1, s6, v2, v1
565; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x80010
566; GFX8-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
567; GFX8-NEXT:    v_mov_b32_e32 v1, s9
568; GFX8-NEXT:    s_ashr_i32 s3, s3, 24
569; GFX8-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
570; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
571; GFX8-NEXT:    v_mov_b32_e32 v1, s3
572; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
573; GFX8-NEXT:    v_mov_b32_e32 v0, s0
574; GFX8-NEXT:    v_mov_b32_e32 v1, s1
575; GFX8-NEXT:    flat_store_dword v[0:1], v2
576; GFX8-NEXT:    s_endpgm
577;
578; GFX9-NODL-LABEL: idot4_multiuse_mul1:
579; GFX9-NODL:       ; %bb.0: ; %entry
580; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
581; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
582; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
583; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
584; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
585; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
586; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
587; GFX9-NODL-NEXT:    s_sext_i32_i8 s4, s2
588; GFX9-NODL-NEXT:    s_sext_i32_i8 s5, s3
589; GFX9-NODL-NEXT:    s_bfe_i32 s7, s3, 0x80008
590; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
591; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s10
592; GFX9-NODL-NEXT:    s_bfe_i32 s6, s2, 0x80008
593; GFX9-NODL-NEXT:    v_mad_i32_i24 v1, s4, v0, v1
594; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s7
595; GFX9-NODL-NEXT:    s_bfe_i32 s9, s3, 0x80010
596; GFX9-NODL-NEXT:    v_mad_i32_i24 v1, s6, v2, v1
597; GFX9-NODL-NEXT:    s_bfe_i32 s8, s2, 0x80010
598; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
599; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s9
600; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 24
601; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
602; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
603; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
604; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
605; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
606; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
607; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
608; GFX9-NODL-NEXT:    s_endpgm
609;
610; GFX9-DL-LABEL: idot4_multiuse_mul1:
611; GFX9-DL:       ; %bb.0: ; %entry
612; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
613; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
614; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
615; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
616; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
617; GFX9-DL-NEXT:    s_load_dword s10, s[0:1], 0x0
618; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
619; GFX9-DL-NEXT:    s_sext_i32_i8 s4, s2
620; GFX9-DL-NEXT:    s_sext_i32_i8 s5, s3
621; GFX9-DL-NEXT:    s_bfe_i32 s7, s3, 0x80008
622; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
623; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s10
624; GFX9-DL-NEXT:    s_bfe_i32 s6, s2, 0x80008
625; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s4, v0, v1
626; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s7
627; GFX9-DL-NEXT:    s_bfe_i32 s9, s3, 0x80010
628; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s6, v2, v1
629; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x80010
630; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
631; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s9
632; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 24
633; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
634; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 24
635; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
636; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v1, v0
637; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
638; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
639; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
640; GFX9-DL-NEXT:    s_endpgm
641;
642; GFX10-DL-LABEL: idot4_multiuse_mul1:
643; GFX10-DL:       ; %bb.0: ; %entry
644; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
645; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
646; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
647; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
648; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
649; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
650; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
651; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
652; GFX10-DL-NEXT:    s_sext_i32_i8 s5, s2
653; GFX10-DL-NEXT:    s_sext_i32_i8 s6, s3
654; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
655; GFX10-DL-NEXT:    s_bfe_i32 s4, s2, 0x80008
656; GFX10-DL-NEXT:    s_bfe_i32 s7, s3, 0x80008
657; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s5, s6, v0
658; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s4, s7, v0
659; GFX10-DL-NEXT:    s_bfe_i32 s4, s2, 0x80010
660; GFX10-DL-NEXT:    s_bfe_i32 s7, s3, 0x80010
661; GFX10-DL-NEXT:    s_ashr_i32 s2, s2, 24
662; GFX10-DL-NEXT:    s_ashr_i32 s3, s3, 24
663; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s5, s6, v0
664; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s4, s7, v0
665; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s3, v0
666; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
667; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
668; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
669; GFX10-DL-NEXT:    s_endpgm
670                                               <4 x i8> addrspace(1)* %src2,
671                                               i32 addrspace(1)* nocapture %dst) {
672entry:
673  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
674  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
675
676  %v1e0 = extractelement <4 x i8> %vec1, i64 0
677  %cv1e0 = sext i8 %v1e0 to i32
678  %v2e0 = extractelement <4 x i8> %vec2, i64 0
679  %cv2e0 = sext i8 %v2e0 to i32
680  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
681
682  %v1e1 = extractelement <4 x i8> %vec1, i64 1
683  %cv1e1 = sext i8 %v1e1 to i32
684  %v2e1 = extractelement <4 x i8> %vec2, i64 1
685  %cv2e1 = sext i8 %v2e1 to i32
686  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
687
688  %v1e2 = extractelement <4 x i8> %vec1, i64 2
689  %cv1e2 = sext i8 %v1e2 to i32
690  %v2e2 = extractelement <4 x i8> %vec2, i64 2
691  %cv2e2 = sext i8 %v2e2 to i32
692  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
693
694  %v1e3 = extractelement <4 x i8> %vec1, i64 3
695  %cv1e3 = sext i8 %v1e3 to i32
696  %v2e3 = extractelement <4 x i8> %vec2, i64 3
697  %cv2e3 = sext i8 %v2e3 to i32
698  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
699
700  %acc = load i32, i32 addrspace(1)* %dst, align 4
701  %add = add i32 %mul1, %acc
702  %add1 = add i32 %mul2, %add
703  %add2 = add i32 %add1, %mul1
704  %add3 = add i32 %add2, %mul3
705  %add4 = add i32 %add3, %mul4
706
707  store i32 %add4, i32 addrspace(1)* %dst, align 4
708  ret void
709}
710
711; TODO: Support this pattern.
712define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
713; GFX7-LABEL: idot4_acc32_vecMul:
714; GFX7:       ; %bb.0: ; %entry
715; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
716; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
717; GFX7-NEXT:    s_mov_b32 s3, 0xf000
718; GFX7-NEXT:    s_mov_b32 s2, -1
719; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
720; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
721; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
722; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
723; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
724; GFX7-NEXT:    s_ashr_i32 s6, s4, 24
725; GFX7-NEXT:    s_ashr_i32 s9, s5, 24
726; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80010
727; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80008
728; GFX7-NEXT:    s_sext_i32_i8 s5, s5
729; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x80010
730; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80008
731; GFX7-NEXT:    s_sext_i32_i8 s4, s4
732; GFX7-NEXT:    v_mov_b32_e32 v0, s5
733; GFX7-NEXT:    v_mov_b32_e32 v1, s12
734; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
735; GFX7-NEXT:    v_mov_b32_e32 v1, s11
736; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
737; GFX7-NEXT:    v_mov_b32_e32 v1, s10
738; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v1, v0
739; GFX7-NEXT:    v_mov_b32_e32 v1, s9
740; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
741; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
742; GFX7-NEXT:    s_endpgm
743;
744; GFX8-LABEL: idot4_acc32_vecMul:
745; GFX8:       ; %bb.0: ; %entry
746; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
747; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
748; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
749; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
750; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
751; GFX8-NEXT:    s_load_dword s8, s[0:1], 0x0
752; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
753; GFX8-NEXT:    v_lshrrev_b16_e64 v0, 8, s2
754; GFX8-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
755; GFX8-NEXT:    s_ashr_i32 s6, s3, 24
756; GFX8-NEXT:    s_bfe_i32 s7, s3, 0x80010
757; GFX8-NEXT:    s_sext_i32_i8 s3, s3
758; GFX8-NEXT:    s_ashr_i32 s4, s2, 24
759; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x80010
760; GFX8-NEXT:    s_sext_i32_i8 s2, s2
761; GFX8-NEXT:    v_mov_b32_e32 v2, s3
762; GFX8-NEXT:    v_mov_b32_e32 v3, s8
763; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 8
764; GFX8-NEXT:    v_bfe_i32 v1, v1, 0, 8
765; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v2, v3
766; GFX8-NEXT:    v_mad_i32_i24 v0, v0, v1, v2
767; GFX8-NEXT:    v_mov_b32_e32 v1, s7
768; GFX8-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
769; GFX8-NEXT:    v_mov_b32_e32 v1, s6
770; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v1, v0
771; GFX8-NEXT:    v_mov_b32_e32 v0, s0
772; GFX8-NEXT:    v_mov_b32_e32 v1, s1
773; GFX8-NEXT:    flat_store_dword v[0:1], v2
774; GFX8-NEXT:    s_endpgm
775;
776; GFX9-NODL-LABEL: idot4_acc32_vecMul:
777; GFX9-NODL:       ; %bb.0: ; %entry
778; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
779; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
780; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
781; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
782; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
783; GFX9-NODL-NEXT:    s_load_dword s8, s[0:1], 0x0
784; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
785; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v0, 8, s2
786; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
787; GFX9-NODL-NEXT:    s_ashr_i32 s6, s3, 24
788; GFX9-NODL-NEXT:    s_bfe_i32 s7, s3, 0x80010
789; GFX9-NODL-NEXT:    s_sext_i32_i8 s3, s3
790; GFX9-NODL-NEXT:    s_ashr_i32 s4, s2, 24
791; GFX9-NODL-NEXT:    s_bfe_i32 s5, s2, 0x80010
792; GFX9-NODL-NEXT:    s_sext_i32_i8 s2, s2
793; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s3
794; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s8
795; GFX9-NODL-NEXT:    v_bfe_i32 v0, v0, 0, 8
796; GFX9-NODL-NEXT:    v_bfe_i32 v1, v1, 0, 8
797; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v2, v3
798; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, v0, v1, v2
799; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s7
800; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
801; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
802; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s4, v1, v0
803; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
804; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
805; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
806; GFX9-NODL-NEXT:    s_endpgm
807;
808; GFX9-DL-LABEL: idot4_acc32_vecMul:
809; GFX9-DL:       ; %bb.0: ; %entry
810; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
811; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
812; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
813; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
814; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
815; GFX9-DL-NEXT:    s_load_dword s8, s[0:1], 0x0
816; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
817; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v0, 8, s2
818; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
819; GFX9-DL-NEXT:    s_ashr_i32 s6, s3, 24
820; GFX9-DL-NEXT:    s_bfe_i32 s7, s3, 0x80010
821; GFX9-DL-NEXT:    s_sext_i32_i8 s3, s3
822; GFX9-DL-NEXT:    s_ashr_i32 s4, s2, 24
823; GFX9-DL-NEXT:    s_bfe_i32 s5, s2, 0x80010
824; GFX9-DL-NEXT:    s_sext_i32_i8 s2, s2
825; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
826; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s8
827; GFX9-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
828; GFX9-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
829; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v2, v3
830; GFX9-DL-NEXT:    v_mad_i32_i24 v0, v0, v1, v2
831; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s7
832; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
833; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
834; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s4, v1, v0
835; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
836; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
837; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
838; GFX9-DL-NEXT:    s_endpgm
839;
840; GFX10-DL-LABEL: idot4_acc32_vecMul:
841; GFX10-DL:       ; %bb.0: ; %entry
842; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
843; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
844; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
845; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
846; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
847; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
848; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
849; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
850; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v0, 8, s2
851; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
852; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
853; GFX10-DL-NEXT:    s_sext_i32_i8 s5, s2
854; GFX10-DL-NEXT:    s_sext_i32_i8 s6, s3
855; GFX10-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
856; GFX10-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
857; GFX10-DL-NEXT:    s_bfe_i32 s4, s2, 0x80010
858; GFX10-DL-NEXT:    s_ashr_i32 s2, s2, 24
859; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s5, s6, v2
860; GFX10-DL-NEXT:    s_bfe_i32 s5, s3, 0x80010
861; GFX10-DL-NEXT:    s_ashr_i32 s3, s3, 24
862; GFX10-DL-NEXT:    v_mad_i32_i24 v0, v0, v1, v2
863; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s4, s5, v0
864; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s2, s3, v0
865; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
866; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
867; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
868; GFX10-DL-NEXT:    s_endpgm
869                                              <4 x i8> addrspace(1)* %src2,
870                                              i32 addrspace(1)* nocapture %dst) {
871entry:
872  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
873  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
874
875  %cvec1 = sext <4 x i8> %vec1 to <4 x i32>
876  %cvec2 = sext <4 x i8> %vec2 to <4 x i32>
877
878  %mul = mul <4 x i32> %cvec1, %cvec2
879  %mul0 = extractelement <4 x i32> %mul, i64 0
880  %mul1 = extractelement <4 x i32> %mul, i64 1
881  %mul2 = extractelement <4 x i32> %mul, i64 2
882  %mul3 = extractelement <4 x i32> %mul, i64 3
883
884  %acc = load i32, i32 addrspace(1)* %dst, align 4
885  %add1 = add i32 %mul0, %acc
886  %add2 = add i32 %add1, %mul1
887  %add3 = add i32 %add2, %mul2
888  %add4 = add i32 %add3, %mul3
889
890  store i32 %add4, i32 addrspace(1)* %dst, align 4
891  ret void
892}
893
894define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
895; GFX7-LABEL: idot4_acc16_vecMul:
896; GFX7:       ; %bb.0: ; %entry
897; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
898; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
899; GFX7-NEXT:    s_mov_b32 s3, 0xf000
900; GFX7-NEXT:    s_mov_b32 s2, -1
901; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
902; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
903; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
904; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
905; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
906; GFX7-NEXT:    s_ashr_i32 s6, s4, 24
907; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80010
908; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80008
909; GFX7-NEXT:    s_ashr_i32 s9, s5, 24
910; GFX7-NEXT:    s_sext_i32_i8 s5, s5
911; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x80010
912; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80008
913; GFX7-NEXT:    s_sext_i32_i8 s4, s4
914; GFX7-NEXT:    v_mov_b32_e32 v1, s5
915; GFX7-NEXT:    v_mov_b32_e32 v2, s11
916; GFX7-NEXT:    v_mov_b32_e32 v3, s10
917; GFX7-NEXT:    s_waitcnt vmcnt(0)
918; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
919; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v2, v0
920; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v3, v0
921; GFX7-NEXT:    v_mov_b32_e32 v1, s9
922; GFX7-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
923; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
924; GFX7-NEXT:    s_endpgm
925;
926; GFX8-LABEL: idot4_acc16_vecMul:
927; GFX8:       ; %bb.0: ; %entry
928; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
929; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
930; GFX8-NEXT:    s_mov_b32 s2, 0xffff
931; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
932; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
933; GFX8-NEXT:    v_mov_b32_e32 v0, s0
934; GFX8-NEXT:    v_mov_b32_e32 v1, s1
935; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
936; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
937; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
938; GFX8-NEXT:    s_bfe_i32 s6, s3, 0x80000
939; GFX8-NEXT:    s_lshr_b32 s4, s3, 16
940; GFX8-NEXT:    v_ashrrev_i16_e64 v3, 8, s3
941; GFX8-NEXT:    s_bfe_i32 s3, s4, 0x80000
942; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
943; GFX8-NEXT:    s_bfe_i32 s5, s0, 0x80000
944; GFX8-NEXT:    v_ashrrev_i16_e64 v4, 8, s0
945; GFX8-NEXT:    s_bfe_i32 s0, s1, 0x80000
946; GFX8-NEXT:    v_ashrrev_i16_e64 v6, 8, s1
947; GFX8-NEXT:    s_and_b32 s1, s2, s6
948; GFX8-NEXT:    v_ashrrev_i16_e64 v5, 8, s4
949; GFX8-NEXT:    s_and_b32 s4, s2, s5
950; GFX8-NEXT:    v_mov_b32_e32 v7, s1
951; GFX8-NEXT:    s_and_b32 s3, s2, s3
952; GFX8-NEXT:    s_and_b32 s0, s2, s0
953; GFX8-NEXT:    s_waitcnt vmcnt(0)
954; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v7, v2
955; GFX8-NEXT:    v_mad_u32_u24 v2, v4, v3, v2
956; GFX8-NEXT:    v_mov_b32_e32 v3, s3
957; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
958; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v5, v2
959; GFX8-NEXT:    flat_store_short v[0:1], v2
960; GFX8-NEXT:    s_endpgm
961;
962; GFX9-NODL-LABEL: idot4_acc16_vecMul:
963; GFX9-NODL:       ; %bb.0: ; %entry
964; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
965; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
966; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, 0xffff
967; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
968; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
969; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
970; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
971; GFX9-NODL-NEXT:    s_lshr_b32 s4, s2, 16
972; GFX9-NODL-NEXT:    s_lshr_b32 s5, s3, 16
973; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v3, 8, s5
974; GFX9-NODL-NEXT:    s_bfe_i32 s5, s5, 0x80000
975; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v2, 8, s4
976; GFX9-NODL-NEXT:    v_and_b32_e32 v5, s5, v4
977; GFX9-NODL-NEXT:    s_bfe_i32 s4, s4, 0x80000
978; GFX9-NODL-NEXT:    v_lshl_or_b32 v3, v3, 16, v5
979; GFX9-NODL-NEXT:    v_and_b32_e32 v5, s4, v4
980; GFX9-NODL-NEXT:    v_lshl_or_b32 v2, v2, 16, v5
981; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v1, 8, s3
982; GFX9-NODL-NEXT:    s_bfe_i32 s3, s3, 0x80000
983; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v0, 8, s2
984; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v2, v2, v3
985; GFX9-NODL-NEXT:    v_and_b32_e32 v3, s3, v4
986; GFX9-NODL-NEXT:    s_bfe_i32 s2, s2, 0x80000
987; GFX9-NODL-NEXT:    v_lshl_or_b32 v1, v1, 16, v3
988; GFX9-NODL-NEXT:    v_and_b32_e32 v3, s2, v4
989; GFX9-NODL-NEXT:    v_lshl_or_b32 v0, v0, 16, v3
990; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v3, v0, v1
991; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
992; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
993; GFX9-NODL-NEXT:    global_load_ushort v4, v[0:1], off
994; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
995; GFX9-NODL-NEXT:    v_add_u32_e32 v4, v3, v4
996; GFX9-NODL-NEXT:    v_add_u32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
997; GFX9-NODL-NEXT:    v_add_u32_e32 v3, v3, v2
998; GFX9-NODL-NEXT:    v_add_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
999; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
1000; GFX9-NODL-NEXT:    s_endpgm
1001;
1002; GFX9-DL-LABEL: idot4_acc16_vecMul:
1003; GFX9-DL:       ; %bb.0: ; %entry
1004; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1005; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1006; GFX9-DL-NEXT:    v_mov_b32_e32 v4, 0xffff
1007; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1008; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1009; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1010; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1011; GFX9-DL-NEXT:    s_lshr_b32 s4, s2, 16
1012; GFX9-DL-NEXT:    s_lshr_b32 s5, s3, 16
1013; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v3, 8, s5
1014; GFX9-DL-NEXT:    s_bfe_i32 s5, s5, 0x80000
1015; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v2, 8, s4
1016; GFX9-DL-NEXT:    v_and_b32_e32 v5, s5, v4
1017; GFX9-DL-NEXT:    s_bfe_i32 s4, s4, 0x80000
1018; GFX9-DL-NEXT:    v_lshl_or_b32 v3, v3, 16, v5
1019; GFX9-DL-NEXT:    v_and_b32_e32 v5, s4, v4
1020; GFX9-DL-NEXT:    v_lshl_or_b32 v2, v2, 16, v5
1021; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v1, 8, s3
1022; GFX9-DL-NEXT:    s_bfe_i32 s3, s3, 0x80000
1023; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v0, 8, s2
1024; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v3
1025; GFX9-DL-NEXT:    v_and_b32_e32 v3, s3, v4
1026; GFX9-DL-NEXT:    s_bfe_i32 s2, s2, 0x80000
1027; GFX9-DL-NEXT:    v_lshl_or_b32 v1, v1, 16, v3
1028; GFX9-DL-NEXT:    v_and_b32_e32 v3, s2, v4
1029; GFX9-DL-NEXT:    v_lshl_or_b32 v0, v0, 16, v3
1030; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, v0, v1
1031; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1032; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1033; GFX9-DL-NEXT:    global_load_ushort v4, v[0:1], off
1034; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
1035; GFX9-DL-NEXT:    v_add_u32_e32 v4, v3, v4
1036; GFX9-DL-NEXT:    v_add_u32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1037; GFX9-DL-NEXT:    v_add_u32_e32 v3, v3, v2
1038; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1039; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
1040; GFX9-DL-NEXT:    s_endpgm
1041;
1042; GFX10-DL-LABEL: idot4_acc16_vecMul:
1043; GFX10-DL:       ; %bb.0: ; %entry
1044; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1045; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1046; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
1047; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1048; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1049; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1050; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1051; GFX10-DL-NEXT:    global_load_ushort v3, v[0:1], off
1052; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
1053; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
1054; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1055; GFX10-DL-NEXT:    s_bfe_i32 s4, s0, 0x80000
1056; GFX10-DL-NEXT:    s_bfe_i32 s3, s1, 0x80000
1057; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 16
1058; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v4, 8, s0
1059; GFX10-DL-NEXT:    s_lshr_b32 s5, s1, 16
1060; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v5, 8, s1
1061; GFX10-DL-NEXT:    v_and_b32_e32 v6, s3, v2
1062; GFX10-DL-NEXT:    v_and_b32_e32 v7, s4, v2
1063; GFX10-DL-NEXT:    s_bfe_i32 s0, s2, 0x80000
1064; GFX10-DL-NEXT:    s_bfe_i32 s1, s5, 0x80000
1065; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v11, 8, s2
1066; GFX10-DL-NEXT:    v_lshl_or_b32 v5, v5, 16, v6
1067; GFX10-DL-NEXT:    v_lshl_or_b32 v4, v4, 16, v7
1068; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v7, 8, s5
1069; GFX10-DL-NEXT:    v_and_b32_e32 v8, s1, v2
1070; GFX10-DL-NEXT:    v_and_b32_e32 v2, s0, v2
1071; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v5
1072; GFX10-DL-NEXT:    v_lshl_or_b32 v5, v7, 16, v8
1073; GFX10-DL-NEXT:    v_lshl_or_b32 v2, v11, 16, v2
1074; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v5
1075; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
1076; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v4, v3
1077; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1078; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v2
1079; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1080; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
1081; GFX10-DL-NEXT:    s_endpgm
1082                                              <4 x i8> addrspace(1)* %src2,
1083                                              i16 addrspace(1)* nocapture %dst) {
1084entry:
1085  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
1086  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
1087
1088  %cvec1 = sext <4 x i8> %vec1 to <4 x i16>
1089  %cvec2 = sext <4 x i8> %vec2 to <4 x i16>
1090
1091  %mul = mul <4 x i16> %cvec1, %cvec2
1092  %mul0 = extractelement <4 x i16> %mul, i64 0
1093  %mul1 = extractelement <4 x i16> %mul, i64 1
1094  %mul2 = extractelement <4 x i16> %mul, i64 2
1095  %mul3 = extractelement <4 x i16> %mul, i64 3
1096
1097  %acc = load i16, i16 addrspace(1)* %dst, align 4
1098  %add1 = add i16 %mul0, %acc
1099  %add2 = add i16 %add1, %mul1
1100  %add3 = add i16 %add2, %mul2
1101  %add4 = add i16 %add3, %mul3
1102
1103  store i16 %add4, i16 addrspace(1)* %dst, align 4
1104  ret void
1105}
1106