1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-NODL %s
5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
8
9define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
10; GFX7-LABEL: udot4_acc32:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
13; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
14; GFX7-NEXT:    s_movk_i32 s8, 0xff
15; GFX7-NEXT:    s_mov_b32 s3, 0xf000
16; GFX7-NEXT:    s_mov_b32 s2, -1
17; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
18; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
19; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
20; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
21; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
22; GFX7-NEXT:    s_and_b32 s6, s4, s8
23; GFX7-NEXT:    s_and_b32 s7, s5, s8
24; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
25; GFX7-NEXT:    v_mov_b32_e32 v0, s7
26; GFX7-NEXT:    v_mov_b32_e32 v1, s12
27; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
28; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
29; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
30; GFX7-NEXT:    v_mov_b32_e32 v1, s9
31; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
32; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
33; GFX7-NEXT:    v_mov_b32_e32 v1, s11
34; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
35; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
36; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
37; GFX7-NEXT:    v_mov_b32_e32 v1, s5
38; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
39; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
40; GFX7-NEXT:    s_endpgm
41;
42; GFX8-LABEL: udot4_acc32:
43; GFX8:       ; %bb.0: ; %entry
44; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
45; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
46; GFX8-NEXT:    s_movk_i32 s2, 0xff
47; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
48; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
49; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
50; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
51; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
52; GFX8-NEXT:    s_and_b32 s5, s3, s2
53; GFX8-NEXT:    s_and_b32 s2, s4, s2
54; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x80008
55; GFX8-NEXT:    v_mov_b32_e32 v0, s2
56; GFX8-NEXT:    v_mov_b32_e32 v1, s10
57; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x80010
58; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
59; GFX8-NEXT:    s_bfe_u32 s6, s3, 0x80008
60; GFX8-NEXT:    v_mov_b32_e32 v1, s7
61; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x80010
62; GFX8-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
63; GFX8-NEXT:    v_mov_b32_e32 v1, s9
64; GFX8-NEXT:    s_lshr_b32 s4, s4, 24
65; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
66; GFX8-NEXT:    s_lshr_b32 s3, s3, 24
67; GFX8-NEXT:    v_mov_b32_e32 v1, s4
68; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v1, v0
69; GFX8-NEXT:    v_mov_b32_e32 v0, s0
70; GFX8-NEXT:    v_mov_b32_e32 v1, s1
71; GFX8-NEXT:    flat_store_dword v[0:1], v2
72; GFX8-NEXT:    s_endpgm
73;
74; GFX9-NODL-LABEL: udot4_acc32:
75; GFX9-NODL:       ; %bb.0: ; %entry
76; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
77; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
78; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
79; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
80; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
81; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
82; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
83; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
84; GFX9-NODL-NEXT:    s_and_b32 s5, s3, s2
85; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
86; GFX9-NODL-NEXT:    s_bfe_u32 s7, s4, 0x80008
87; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s2
88; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s10
89; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
90; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
91; GFX9-NODL-NEXT:    s_bfe_u32 s6, s3, 0x80008
92; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s7
93; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
94; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
95; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s9
96; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
97; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
98; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
99; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s4
100; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s3, v1, v0
101; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
102; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
103; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
104; GFX9-NODL-NEXT:    s_endpgm
105;
106; GFX9-DL-LABEL: udot4_acc32:
107; GFX9-DL:       ; %bb.0: ; %entry
108; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
109; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
110; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
111; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
112; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
113; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
114; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
115; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
116; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
117; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
118; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
119; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s4, v2, v3
120; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
121; GFX9-DL-NEXT:    s_endpgm
122;
123; GFX10-DL-LABEL: udot4_acc32:
124; GFX10-DL:       ; %bb.0: ; %entry
125; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
126; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
127; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
128; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
129; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
130; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
131; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
132; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
133; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
134; GFX10-DL-NEXT:    v_dot4_u32_u8 v2, s3, s4, v0
135; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
136; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
137; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
138; GFX10-DL-NEXT:    s_endpgm
139                                       <4 x i8> addrspace(1)* %src2,
140                                       i32 addrspace(1)* nocapture %dst) {
141entry:
142  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
143  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
144
145  %v1e0 = extractelement <4 x i8> %vec1, i64 0
146  %cv1e0 = zext i8 %v1e0 to i32
147  %v2e0 = extractelement <4 x i8> %vec2, i64 0
148  %cv2e0 = zext i8 %v2e0 to i32
149  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
150
151  %v1e1 = extractelement <4 x i8> %vec1, i64 1
152  %cv1e1 = zext i8 %v1e1 to i32
153  %v2e1 = extractelement <4 x i8> %vec2, i64 1
154  %cv2e1 = zext i8 %v2e1 to i32
155  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
156
157  %v1e2 = extractelement <4 x i8> %vec1, i64 2
158  %cv1e2 = zext i8 %v1e2 to i32
159  %v2e2 = extractelement <4 x i8> %vec2, i64 2
160  %cv2e2 = zext i8 %v2e2 to i32
161  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
162
163  %v1e3 = extractelement <4 x i8> %vec1, i64 3
164  %cv1e3 = zext i8 %v1e3 to i32
165  %v2e3 = extractelement <4 x i8> %vec2, i64 3
166  %cv2e3 = zext i8 %v2e3 to i32
167  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
168
169  %acc = load i32, i32 addrspace(1)* %dst, align 4
170  %mad1 = add i32 %mul1, %acc
171  %mad2 = add i32 %mad1, %mul2
172  %mad3 = add i32 %mad2, %mul3
173  %mad4 = add i32 %mad3, %mul4
174
175  store i32 %mad4, i32 addrspace(1)* %dst, align 4
176  ret void
177}
178
179define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1,
180; GFX7-LABEL: udot4_acc16:
181; GFX7:       ; %bb.0: ; %entry
182; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
183; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
184; GFX7-NEXT:    s_mov_b32 s3, 0xf000
185; GFX7-NEXT:    s_mov_b32 s2, -1
186; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
187; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
188; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
189; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
190; GFX7-NEXT:    s_movk_i32 s5, 0xff
191; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
192; GFX7-NEXT:    s_and_b32 s7, s6, s5
193; GFX7-NEXT:    s_bfe_u32 s8, s6, 0x80008
194; GFX7-NEXT:    s_and_b32 s5, s4, s5
195; GFX7-NEXT:    v_mov_b32_e32 v1, s7
196; GFX7-NEXT:    s_bfe_u32 s10, s6, 0x80010
197; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
198; GFX7-NEXT:    v_mov_b32_e32 v2, s8
199; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
200; GFX7-NEXT:    s_lshr_b32 s6, s6, 24
201; GFX7-NEXT:    v_mov_b32_e32 v3, s10
202; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
203; GFX7-NEXT:    s_waitcnt vmcnt(0)
204; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
205; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
206; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
207; GFX7-NEXT:    v_mov_b32_e32 v1, s6
208; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
209; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
210; GFX7-NEXT:    s_endpgm
211;
212; GFX8-LABEL: udot4_acc16:
213; GFX8:       ; %bb.0: ; %entry
214; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
215; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
216; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
217; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
218; GFX8-NEXT:    v_mov_b32_e32 v0, s0
219; GFX8-NEXT:    v_mov_b32_e32 v1, s1
220; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
221; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
222; GFX8-NEXT:    s_movk_i32 s0, 0xff
223; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
224; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
225; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
226; GFX8-NEXT:    v_mov_b32_e32 v4, s5
227; GFX8-NEXT:    s_and_b32 s3, s1, s0
228; GFX8-NEXT:    s_and_b32 s0, s2, s0
229; GFX8-NEXT:    v_mov_b32_e32 v3, s0
230; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x80008
231; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
232; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
233; GFX8-NEXT:    v_mov_b32_e32 v5, s7
234; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
235; GFX8-NEXT:    s_waitcnt vmcnt(0)
236; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
237; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v4, v2
238; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
239; GFX8-NEXT:    v_mov_b32_e32 v3, s2
240; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
241; GFX8-NEXT:    flat_store_short v[0:1], v2
242; GFX8-NEXT:    s_endpgm
243;
244; GFX9-NODL-LABEL: udot4_acc16:
245; GFX9-NODL:       ; %bb.0: ; %entry
246; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
247; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
248; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
249; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
250; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
251; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
252; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
253; GFX9-NODL-NEXT:    s_load_dword s1, s[4:5], 0x0
254; GFX9-NODL-NEXT:    s_movk_i32 s0, 0xff
255; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
256; GFX9-NODL-NEXT:    s_bfe_u32 s5, s2, 0x80008
257; GFX9-NODL-NEXT:    s_bfe_u32 s7, s2, 0x80010
258; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s5
259; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s0
260; GFX9-NODL-NEXT:    s_and_b32 s0, s2, s0
261; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
262; GFX9-NODL-NEXT:    s_bfe_u32 s4, s1, 0x80008
263; GFX9-NODL-NEXT:    s_bfe_u32 s6, s1, 0x80010
264; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
265; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s7
266; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 24
267; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
268; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
269; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, v4, v2
270; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
271; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
272; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
273; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
274; GFX9-NODL-NEXT:    s_endpgm
275;
276; GFX9-DL-LABEL: udot4_acc16:
277; GFX9-DL:       ; %bb.0: ; %entry
278; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
279; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
280; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
281; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
282; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
283; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
284; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
285; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
286; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
287; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
288; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
289; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s0, v3, v2
290; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
291; GFX9-DL-NEXT:    s_endpgm
292;
293; GFX10-DL-LABEL: udot4_acc16:
294; GFX10-DL:       ; %bb.0: ; %entry
295; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
296; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
297; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
298; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
299; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s3
300; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
301; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
302; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
303; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
304; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
305; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
306; GFX10-DL-NEXT:    v_dot4_u32_u8 v2, s0, s1, v2
307; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
308; GFX10-DL-NEXT:    s_endpgm
309                                       <4 x i8> addrspace(1)* %src2,
310                                       i16 addrspace(1)* nocapture %dst) {
311entry:
312  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
313  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
314
315  %v1e0 = extractelement <4 x i8> %vec1, i64 0
316  %cv1e0 = zext i8 %v1e0 to i16
317  %v2e0 = extractelement <4 x i8> %vec2, i64 0
318  %cv2e0 = zext i8 %v2e0 to i16
319  %mul1 = mul nuw nsw i16 %cv1e0, %cv2e0
320
321  %v1e1 = extractelement <4 x i8> %vec1, i64 1
322  %cv1e1 = zext i8 %v1e1 to i16
323  %v2e1 = extractelement <4 x i8> %vec2, i64 1
324  %cv2e1 = zext i8 %v2e1 to i16
325  %mul2 = mul nuw nsw i16 %cv1e1, %cv2e1
326
327  %v1e2 = extractelement <4 x i8> %vec1, i64 2
328  %cv1e2 = zext i8 %v1e2 to i16
329  %v2e2 = extractelement <4 x i8> %vec2, i64 2
330  %cv2e2 = zext i8 %v2e2 to i16
331  %mul3 = mul nuw nsw i16 %cv1e2, %cv2e2
332
333  %v1e3 = extractelement <4 x i8> %vec1, i64 3
334  %cv1e3 = zext i8 %v1e3 to i16
335  %v2e3 = extractelement <4 x i8> %vec2, i64 3
336  %cv2e3 = zext i8 %v2e3 to i16
337  %mul4 = mul nuw nsw i16 %cv1e3, %cv2e3
338
339  %acc = load i16, i16 addrspace(1)* %dst, align 2
340  %mad1 = add i16 %mul1, %acc
341  %mad2 = add i16 %mad1, %mul2
342  %mad3 = add i16 %mad2, %mul3
343  %mad4 = add i16 %mad3, %mul4
344
345  store i16 %mad4, i16 addrspace(1)* %dst, align 2
346  ret void
347}
348
349define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1,
350; GFX7-LABEL: udot4_acc8:
351; GFX7:       ; %bb.0: ; %entry
352; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
353; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
354; GFX7-NEXT:    s_mov_b32 s3, 0xf000
355; GFX7-NEXT:    s_mov_b32 s2, -1
356; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
357; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
358; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
359; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
360; GFX7-NEXT:    s_movk_i32 s5, 0xff
361; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
362; GFX7-NEXT:    s_and_b32 s7, s6, s5
363; GFX7-NEXT:    s_bfe_u32 s8, s6, 0x80008
364; GFX7-NEXT:    s_and_b32 s5, s4, s5
365; GFX7-NEXT:    v_mov_b32_e32 v1, s7
366; GFX7-NEXT:    s_bfe_u32 s10, s6, 0x80010
367; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
368; GFX7-NEXT:    v_mov_b32_e32 v2, s8
369; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
370; GFX7-NEXT:    s_lshr_b32 s6, s6, 24
371; GFX7-NEXT:    v_mov_b32_e32 v3, s10
372; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
373; GFX7-NEXT:    s_waitcnt vmcnt(0)
374; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
375; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
376; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
377; GFX7-NEXT:    v_mov_b32_e32 v1, s6
378; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
379; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
380; GFX7-NEXT:    s_endpgm
381;
382; GFX8-LABEL: udot4_acc8:
383; GFX8:       ; %bb.0: ; %entry
384; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
385; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
386; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
387; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
388; GFX8-NEXT:    v_mov_b32_e32 v0, s0
389; GFX8-NEXT:    v_mov_b32_e32 v1, s1
390; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
391; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
392; GFX8-NEXT:    s_movk_i32 s0, 0xff
393; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
394; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
395; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
396; GFX8-NEXT:    s_and_b32 s3, s1, s0
397; GFX8-NEXT:    s_and_b32 s0, s2, s0
398; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x80008
399; GFX8-NEXT:    v_mov_b32_e32 v3, s3
400; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
401; GFX8-NEXT:    v_mov_b32_e32 v4, s4
402; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
403; GFX8-NEXT:    v_mov_b32_e32 v5, s6
404; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
405; GFX8-NEXT:    s_waitcnt vmcnt(0)
406; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
407; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
408; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
409; GFX8-NEXT:    v_mov_b32_e32 v3, s1
410; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
411; GFX8-NEXT:    flat_store_byte v[0:1], v2
412; GFX8-NEXT:    s_endpgm
413;
414; GFX9-NODL-LABEL: udot4_acc8:
415; GFX9-NODL:       ; %bb.0: ; %entry
416; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
417; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
418; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
419; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
420; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
421; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
422; GFX9-NODL-NEXT:    global_load_ubyte v2, v[0:1], off
423; GFX9-NODL-NEXT:    s_load_dword s1, s[6:7], 0x0
424; GFX9-NODL-NEXT:    s_movk_i32 s0, 0xff
425; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
426; GFX9-NODL-NEXT:    s_bfe_u32 s5, s2, 0x80008
427; GFX9-NODL-NEXT:    s_bfe_u32 s7, s2, 0x80010
428; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s0
429; GFX9-NODL-NEXT:    s_and_b32 s0, s2, s0
430; GFX9-NODL-NEXT:    s_bfe_u32 s4, s1, 0x80008
431; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
432; GFX9-NODL-NEXT:    s_bfe_u32 s6, s1, 0x80010
433; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s4
434; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 24
435; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s6
436; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
437; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
438; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
439; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
440; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
441; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
442; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
443; GFX9-NODL-NEXT:    global_store_byte v[0:1], v2, off
444; GFX9-NODL-NEXT:    s_endpgm
445;
446; GFX9-DL-LABEL: udot4_acc8:
447; GFX9-DL:       ; %bb.0: ; %entry
448; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
449; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
450; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
451; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
452; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
453; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
454; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
455; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
456; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
457; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
458; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
459; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s0, v3, v2
460; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
461; GFX9-DL-NEXT:    s_endpgm
462;
463; GFX10-DL-LABEL: udot4_acc8:
464; GFX10-DL:       ; %bb.0: ; %entry
465; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
466; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
467; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
468; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
469; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s3
470; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
471; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
472; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
473; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
474; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
475; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
476; GFX10-DL-NEXT:    v_dot4_u32_u8 v2, s0, s1, v2
477; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
478; GFX10-DL-NEXT:    s_endpgm
479                                      <4 x i8> addrspace(1)* %src2,
480                                      i8 addrspace(1)* nocapture %dst) {
481entry:
482  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
483  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
484
485  %v1e0 = extractelement <4 x i8> %vec1, i64 0
486  %v2e0 = extractelement <4 x i8> %vec2, i64 0
487  %mul1 = mul nuw nsw i8 %v1e0, %v2e0
488
489  %v1e1 = extractelement <4 x i8> %vec1, i64 1
490  %v2e1 = extractelement <4 x i8> %vec2, i64 1
491  %mul2 = mul nuw nsw i8 %v1e1, %v2e1
492
493  %v1e2 = extractelement <4 x i8> %vec1, i64 2
494  %v2e2 = extractelement <4 x i8> %vec2, i64 2
495  %mul3 = mul nuw nsw i8 %v1e2, %v2e2
496
497  %v1e3 = extractelement <4 x i8> %vec1, i64 3
498  %v2e3 = extractelement <4 x i8> %vec2, i64 3
499  %mul4 = mul nuw nsw i8 %v1e3, %v2e3
500
501  %acc = load i8, i8 addrspace(1)* %dst, align 2
502  %mad1 = add i8 %mul1, %acc
503  %mad2 = add i8 %mad1, %mul2
504  %mad3 = add i8 %mad2, %mul3
505  %mad4 = add i8 %mad3, %mul4
506
507  store i8 %mad4, i8 addrspace(1)* %dst, align 2
508  ret void
509}
510
511; TODO: Generate udot4?
512define amdgpu_kernel void @udot2_8(<4 x i8> addrspace(1)* %src1,
513; GFX7-LABEL: udot2_8:
514; GFX7:       ; %bb.0: ; %entry
515; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
516; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
517; GFX7-NEXT:    s_mov_b32 s3, 0xf000
518; GFX7-NEXT:    s_mov_b32 s2, -1
519; GFX7-NEXT:    s_movk_i32 s8, 0xff
520; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
521; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
522; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
523; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
524; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
525; GFX7-NEXT:    s_and_b32 s7, s4, s8
526; GFX7-NEXT:    s_and_b32 s6, s5, s8
527; GFX7-NEXT:    v_mov_b32_e32 v1, s6
528; GFX7-NEXT:    s_bfe_u32 s5, s5, 0x80008
529; GFX7-NEXT:    s_bfe_u32 s4, s4, 0x80008
530; GFX7-NEXT:    s_waitcnt vmcnt(0)
531; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
532; GFX7-NEXT:    v_mov_b32_e32 v1, s5
533; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
534; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
535; GFX7-NEXT:    s_endpgm
536;
537; GFX8-LABEL: udot2_8:
538; GFX8:       ; %bb.0: ; %entry
539; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
540; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
541; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
542; GFX8-NEXT:    v_mov_b32_e32 v0, s0
543; GFX8-NEXT:    v_mov_b32_e32 v1, s1
544; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
545; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
546; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
547; GFX8-NEXT:    s_movk_i32 s0, 0xff
548; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
549; GFX8-NEXT:    s_and_b32 s3, s2, s0
550; GFX8-NEXT:    s_and_b32 s0, s1, s0
551; GFX8-NEXT:    v_mov_b32_e32 v3, s3
552; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x80008
553; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x80008
554; GFX8-NEXT:    s_waitcnt vmcnt(0)
555; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
556; GFX8-NEXT:    v_mov_b32_e32 v3, s2
557; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
558; GFX8-NEXT:    flat_store_byte v[0:1], v2
559; GFX8-NEXT:    s_endpgm
560;
561; GFX9-NODL-LABEL: udot2_8:
562; GFX9-NODL:       ; %bb.0: ; %entry
563; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
564; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
565; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
566; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
567; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
568; GFX9-NODL-NEXT:    global_load_ubyte v2, v[0:1], off
569; GFX9-NODL-NEXT:    s_load_dword s1, s[4:5], 0x0
570; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
571; GFX9-NODL-NEXT:    s_movk_i32 s0, 0xff
572; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
573; GFX9-NODL-NEXT:    s_and_b32 s3, s2, s0
574; GFX9-NODL-NEXT:    s_and_b32 s0, s1, s0
575; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
576; GFX9-NODL-NEXT:    s_bfe_u32 s2, s2, 0x80008
577; GFX9-NODL-NEXT:    s_bfe_u32 s1, s1, 0x80008
578; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
579; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
580; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
581; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
582; GFX9-NODL-NEXT:    global_store_byte v[0:1], v2, off
583; GFX9-NODL-NEXT:    s_endpgm
584;
585; GFX9-DL-LABEL: udot2_8:
586; GFX9-DL:       ; %bb.0: ; %entry
587; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
588; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
589; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
590; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
591; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
592; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
593; GFX9-DL-NEXT:    s_load_dword s1, s[4:5], 0x0
594; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
595; GFX9-DL-NEXT:    s_movk_i32 s0, 0xff
596; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
597; GFX9-DL-NEXT:    s_and_b32 s3, s2, s0
598; GFX9-DL-NEXT:    s_and_b32 s0, s1, s0
599; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
600; GFX9-DL-NEXT:    s_bfe_u32 s2, s2, 0x80008
601; GFX9-DL-NEXT:    s_bfe_u32 s1, s1, 0x80008
602; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
603; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
604; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
605; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
606; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
607; GFX9-DL-NEXT:    s_endpgm
608;
609; GFX10-DL-LABEL: udot2_8:
610; GFX10-DL:       ; %bb.0: ; %entry
611; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
612; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
613; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
614; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
615; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
616; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
617; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
618; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
619; GFX10-DL-NEXT:    s_load_dword s0, s[6:7], 0x0
620; GFX10-DL-NEXT:    s_load_dword s1, s[4:5], 0x0
621; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
622; GFX10-DL-NEXT:    s_and_b32 s3, s0, s2
623; GFX10-DL-NEXT:    s_and_b32 s2, s1, s2
624; GFX10-DL-NEXT:    s_bfe_u32 s0, s0, 0x80008
625; GFX10-DL-NEXT:    s_bfe_u32 s1, s1, 0x80008
626; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
627; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v2
628; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
629; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
630; GFX10-DL-NEXT:    s_endpgm
631                                   <4 x i8> addrspace(1)* %src2,
632                                   i8 addrspace(1)* nocapture %dst) {
633entry:
634  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
635  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
636
637  %v1e0 = extractelement <4 x i8> %vec1, i64 0
638  %v2e0 = extractelement <4 x i8> %vec2, i64 0
639  %mul1 = mul nuw nsw i8 %v1e0, %v2e0
640
641  %v1e1 = extractelement <4 x i8> %vec1, i64 1
642  %v2e1 = extractelement <4 x i8> %vec2, i64 1
643  %mul2 = mul nuw nsw i8 %v1e1, %v2e1
644
645  %acc = load i8, i8 addrspace(1)* %dst, align 2
646  %mad1 = add i8 %mul1, %acc
647  %mad2 = add i8 %mad1, %mul2
648  store i8 %mad2, i8 addrspace(1)* %dst, align 2
649  ret void
650}
651
652define amdgpu_kernel void @udot4_CommutationInsideMAD(<4 x i8> addrspace(1)* %src1,
653; GFX7-LABEL: udot4_CommutationInsideMAD:
654; GFX7:       ; %bb.0: ; %entry
655; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
656; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
657; GFX7-NEXT:    s_mov_b32 s3, 0xf000
658; GFX7-NEXT:    s_mov_b32 s2, -1
659; GFX7-NEXT:    s_movk_i32 s8, 0xff
660; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
661; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
662; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
663; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
664; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
665; GFX7-NEXT:    s_and_b32 s6, s4, s8
666; GFX7-NEXT:    s_and_b32 s7, s5, s8
667; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
668; GFX7-NEXT:    v_mov_b32_e32 v1, s6
669; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
670; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
671; GFX7-NEXT:    v_mov_b32_e32 v2, s8
672; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
673; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
674; GFX7-NEXT:    v_mov_b32_e32 v3, s10
675; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
676; GFX7-NEXT:    s_waitcnt vmcnt(0)
677; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
678; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
679; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
680; GFX7-NEXT:    v_mov_b32_e32 v1, s4
681; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
682; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
683; GFX7-NEXT:    s_endpgm
684;
685; GFX8-LABEL: udot4_CommutationInsideMAD:
686; GFX8:       ; %bb.0: ; %entry
687; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
688; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
689; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
690; GFX8-NEXT:    v_mov_b32_e32 v0, s0
691; GFX8-NEXT:    v_mov_b32_e32 v1, s1
692; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
693; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
694; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
695; GFX8-NEXT:    s_movk_i32 s0, 0xff
696; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
697; GFX8-NEXT:    s_and_b32 s3, s1, s0
698; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x80008
699; GFX8-NEXT:    s_and_b32 s0, s2, s0
700; GFX8-NEXT:    v_mov_b32_e32 v3, s3
701; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
702; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
703; GFX8-NEXT:    v_mov_b32_e32 v4, s4
704; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
705; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
706; GFX8-NEXT:    v_mov_b32_e32 v5, s6
707; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
708; GFX8-NEXT:    s_waitcnt vmcnt(0)
709; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
710; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
711; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
712; GFX8-NEXT:    v_mov_b32_e32 v3, s1
713; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
714; GFX8-NEXT:    flat_store_byte v[0:1], v2
715; GFX8-NEXT:    s_endpgm
716;
717; GFX9-NODL-LABEL: udot4_CommutationInsideMAD:
718; GFX9-NODL:       ; %bb.0: ; %entry
719; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
720; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
721; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
722; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
723; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
724; GFX9-NODL-NEXT:    global_load_ubyte v2, v[0:1], off
725; GFX9-NODL-NEXT:    s_load_dword s1, s[4:5], 0x0
726; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
727; GFX9-NODL-NEXT:    s_movk_i32 s0, 0xff
728; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
729; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s0
730; GFX9-NODL-NEXT:    s_bfe_u32 s4, s1, 0x80008
731; GFX9-NODL-NEXT:    s_and_b32 s0, s2, s0
732; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
733; GFX9-NODL-NEXT:    s_bfe_u32 s6, s1, 0x80010
734; GFX9-NODL-NEXT:    s_bfe_u32 s5, s2, 0x80008
735; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s4
736; GFX9-NODL-NEXT:    s_bfe_u32 s7, s2, 0x80010
737; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 24
738; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s6
739; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
740; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
741; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
742; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
743; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
744; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
745; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
746; GFX9-NODL-NEXT:    global_store_byte v[0:1], v2, off
747; GFX9-NODL-NEXT:    s_endpgm
748;
749; GFX9-DL-LABEL: udot4_CommutationInsideMAD:
750; GFX9-DL:       ; %bb.0: ; %entry
751; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
752; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
753; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
754; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
755; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
756; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
757; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
758; GFX9-DL-NEXT:    s_load_dword s0, s[6:7], 0x0
759; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
760; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
761; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
762; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s0, v3, v2
763; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
764; GFX9-DL-NEXT:    s_endpgm
765;
766; GFX10-DL-LABEL: udot4_CommutationInsideMAD:
767; GFX10-DL:       ; %bb.0: ; %entry
768; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
769; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
770; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
771; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
772; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s3
773; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
774; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
775; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
776; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
777; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
778; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
779; GFX10-DL-NEXT:    v_dot4_u32_u8 v2, s1, s0, v2
780; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
781; GFX10-DL-NEXT:    s_endpgm
782                                                      <4 x i8> addrspace(1)* %src2,
783                                                      i8 addrspace(1)* nocapture %dst) {
784entry:
785  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
786  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
787
788  %v1e0 = extractelement <4 x i8> %vec1, i64 0
789  %v2e0 = extractelement <4 x i8> %vec2, i64 0
790  %mul1 = mul nuw nsw i8 %v2e0, %v1e0
791
792  %v1e1 = extractelement <4 x i8> %vec1, i64 1
793  %v2e1 = extractelement <4 x i8> %vec2, i64 1
794  %mul2 = mul nuw nsw i8 %v2e1, %v1e1
795
796  %v1e2 = extractelement <4 x i8> %vec1, i64 2
797  %v2e2 = extractelement <4 x i8> %vec2, i64 2
798  %mul3 = mul nuw nsw i8 %v2e2, %v1e2
799
800  %v1e3 = extractelement <4 x i8> %vec1, i64 3
801  %v2e3 = extractelement <4 x i8> %vec2, i64 3
802  %mul4 = mul nuw nsw i8 %v2e3, %v1e3
803
804  %acc = load i8, i8 addrspace(1)* %dst, align 2
805  %mad1 = add i8 %acc, %mul1
806  %mad2 = add i8 %mul2, %mad1
807  %mad3 = add i8 %mul3, %mad2
808  %mad4 = add i8 %mul4, %mad3
809
810  store i8 %mad4, i8 addrspace(1)* %dst, align 2
811  ret void
812}
813
814; TODO: Support commutation accross the adds.
815define amdgpu_kernel void @udot4_CommutationAccrossMADs(<4 x i8> addrspace(1)* %src1,
816; GFX7-LABEL: udot4_CommutationAccrossMADs:
817; GFX7:       ; %bb.0: ; %entry
818; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
819; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
820; GFX7-NEXT:    s_mov_b32 s3, 0xf000
821; GFX7-NEXT:    s_mov_b32 s2, -1
822; GFX7-NEXT:    s_movk_i32 s8, 0xff
823; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
824; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
825; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
826; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
827; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
828; GFX7-NEXT:    s_and_b32 s6, s4, s8
829; GFX7-NEXT:    s_and_b32 s7, s5, s8
830; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
831; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
832; GFX7-NEXT:    v_mov_b32_e32 v1, s8
833; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
834; GFX7-NEXT:    v_mov_b32_e32 v2, s6
835; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
836; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
837; GFX7-NEXT:    v_mov_b32_e32 v3, s10
838; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
839; GFX7-NEXT:    s_waitcnt vmcnt(0)
840; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
841; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v2, v0
842; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
843; GFX7-NEXT:    v_mov_b32_e32 v1, s4
844; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
845; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
846; GFX7-NEXT:    s_endpgm
847;
848; GFX8-LABEL: udot4_CommutationAccrossMADs:
849; GFX8:       ; %bb.0: ; %entry
850; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
851; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
852; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
853; GFX8-NEXT:    v_mov_b32_e32 v0, s0
854; GFX8-NEXT:    v_mov_b32_e32 v1, s1
855; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
856; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
857; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
858; GFX8-NEXT:    s_movk_i32 s0, 0xff
859; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
860; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x80008
861; GFX8-NEXT:    s_and_b32 s3, s1, s0
862; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
863; GFX8-NEXT:    v_mov_b32_e32 v3, s4
864; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
865; GFX8-NEXT:    s_and_b32 s0, s2, s0
866; GFX8-NEXT:    v_mov_b32_e32 v4, s3
867; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
868; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
869; GFX8-NEXT:    v_mov_b32_e32 v5, s6
870; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
871; GFX8-NEXT:    s_waitcnt vmcnt(0)
872; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v3, v2
873; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v4, v2
874; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
875; GFX8-NEXT:    v_mov_b32_e32 v3, s1
876; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
877; GFX8-NEXT:    flat_store_byte v[0:1], v2
878; GFX8-NEXT:    s_endpgm
879;
880; GFX9-NODL-LABEL: udot4_CommutationAccrossMADs:
881; GFX9-NODL:       ; %bb.0: ; %entry
882; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
883; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
884; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
885; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
886; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
887; GFX9-NODL-NEXT:    global_load_ubyte v2, v[0:1], off
888; GFX9-NODL-NEXT:    s_load_dword s1, s[4:5], 0x0
889; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
890; GFX9-NODL-NEXT:    s_movk_i32 s0, 0xff
891; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
892; GFX9-NODL-NEXT:    s_bfe_u32 s4, s1, 0x80008
893; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s0
894; GFX9-NODL-NEXT:    s_bfe_u32 s5, s2, 0x80008
895; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s4
896; GFX9-NODL-NEXT:    s_bfe_u32 s6, s1, 0x80010
897; GFX9-NODL-NEXT:    s_and_b32 s0, s2, s0
898; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s3
899; GFX9-NODL-NEXT:    s_bfe_u32 s7, s2, 0x80010
900; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 24
901; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s6
902; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
903; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
904; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s5, v3, v2
905; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v4, v2
906; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
907; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
908; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
909; GFX9-NODL-NEXT:    global_store_byte v[0:1], v2, off
910; GFX9-NODL-NEXT:    s_endpgm
911;
912; GFX9-DL-LABEL: udot4_CommutationAccrossMADs:
913; GFX9-DL:       ; %bb.0: ; %entry
914; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
915; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
916; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
917; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
918; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
919; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
920; GFX9-DL-NEXT:    s_load_dword s1, s[4:5], 0x0
921; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
922; GFX9-DL-NEXT:    s_movk_i32 s0, 0xff
923; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
924; GFX9-DL-NEXT:    s_bfe_u32 s4, s1, 0x80008
925; GFX9-DL-NEXT:    s_and_b32 s3, s1, s0
926; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x80008
927; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
928; GFX9-DL-NEXT:    s_bfe_u32 s6, s1, 0x80010
929; GFX9-DL-NEXT:    s_and_b32 s0, s2, s0
930; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s3
931; GFX9-DL-NEXT:    s_bfe_u32 s7, s2, 0x80010
932; GFX9-DL-NEXT:    s_lshr_b32 s1, s1, 24
933; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s6
934; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 24
935; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
936; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v3, v2
937; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v4, v2
938; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
939; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
940; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
941; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
942; GFX9-DL-NEXT:    s_endpgm
943;
944; GFX10-DL-LABEL: udot4_CommutationAccrossMADs:
945; GFX10-DL:       ; %bb.0: ; %entry
946; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
947; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
948; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
949; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
950; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
951; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
952; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
953; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
954; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
955; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
956; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
957; GFX10-DL-NEXT:    s_bfe_u32 s3, s0, 0x80008
958; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x80008
959; GFX10-DL-NEXT:    s_and_b32 s5, s0, s2
960; GFX10-DL-NEXT:    s_and_b32 s2, s1, s2
961; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
962; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s4, s3, v2
963; GFX10-DL-NEXT:    s_bfe_u32 s3, s0, 0x80010
964; GFX10-DL-NEXT:    s_bfe_u32 s4, s1, 0x80010
965; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 24
966; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 24
967; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s5, v2
968; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s4, s3, v2
969; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
970; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
971; GFX10-DL-NEXT:    s_endpgm
972                                                        <4 x i8> addrspace(1)* %src2,
973                                                        i8 addrspace(1)* nocapture %dst) {
974entry:
975  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
976  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
977
978  %v1e0 = extractelement <4 x i8> %vec1, i64 0
979  %v2e0 = extractelement <4 x i8> %vec2, i64 0
980  %mul1 = mul nuw nsw i8 %v2e0, %v1e0
981
982  %v1e1 = extractelement <4 x i8> %vec1, i64 1
983  %v2e1 = extractelement <4 x i8> %vec2, i64 1
984  %mul2 = mul nuw nsw i8 %v2e1, %v1e1
985
986  %v1e2 = extractelement <4 x i8> %vec1, i64 2
987  %v2e2 = extractelement <4 x i8> %vec2, i64 2
988  %mul3 = mul nuw nsw i8 %v2e2, %v1e2
989
990  %v1e3 = extractelement <4 x i8> %vec1, i64 3
991  %v2e3 = extractelement <4 x i8> %vec2, i64 3
992  %mul4 = mul nuw nsw i8 %v2e3, %v1e3
993
994  %acc = load i8, i8 addrspace(1)* %dst, align 2
995  %mad1 = add i8 %acc, %mul2
996  %mad2 = add i8 %mad1, %mul1
997  %mad3 = add i8 %mad2, %mul3
998  %mad4 = add i8 %mad3, %mul4
999
1000  store i8 %mad4, i8 addrspace(1)* %dst, align 2
1001  ret void
1002}
1003
1004define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
1005; GFX7-LABEL: udot4_multiuse_mul1:
1006; GFX7:       ; %bb.0: ; %entry
1007; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1008; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1009; GFX7-NEXT:    s_movk_i32 s8, 0xff
1010; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1011; GFX7-NEXT:    s_mov_b32 s2, -1
1012; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1013; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1014; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1015; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
1016; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1017; GFX7-NEXT:    s_and_b32 s6, s4, s8
1018; GFX7-NEXT:    s_and_b32 s7, s5, s8
1019; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
1020; GFX7-NEXT:    v_mov_b32_e32 v0, s7
1021; GFX7-NEXT:    v_mov_b32_e32 v1, s12
1022; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
1023; GFX7-NEXT:    v_mad_u32_u24 v1, s6, v0, v1
1024; GFX7-NEXT:    v_mov_b32_e32 v2, s9
1025; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
1026; GFX7-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
1027; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
1028; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
1029; GFX7-NEXT:    v_mov_b32_e32 v1, s11
1030; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
1031; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
1032; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
1033; GFX7-NEXT:    v_mov_b32_e32 v1, s5
1034; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1035; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1036; GFX7-NEXT:    s_endpgm
1037;
1038; GFX8-LABEL: udot4_multiuse_mul1:
1039; GFX8:       ; %bb.0: ; %entry
1040; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1041; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1042; GFX8-NEXT:    s_movk_i32 s2, 0xff
1043; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1044; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1045; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1046; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
1047; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1048; GFX8-NEXT:    s_and_b32 s5, s3, s2
1049; GFX8-NEXT:    s_and_b32 s2, s4, s2
1050; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x80008
1051; GFX8-NEXT:    v_mov_b32_e32 v0, s2
1052; GFX8-NEXT:    v_mov_b32_e32 v1, s10
1053; GFX8-NEXT:    s_bfe_u32 s6, s3, 0x80008
1054; GFX8-NEXT:    v_mad_u32_u24 v1, s5, v0, v1
1055; GFX8-NEXT:    v_mov_b32_e32 v2, s7
1056; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x80010
1057; GFX8-NEXT:    v_mad_u32_u24 v1, s6, v2, v1
1058; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x80010
1059; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1060; GFX8-NEXT:    v_mov_b32_e32 v1, s9
1061; GFX8-NEXT:    s_lshr_b32 s4, s4, 24
1062; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
1063; GFX8-NEXT:    s_lshr_b32 s3, s3, 24
1064; GFX8-NEXT:    v_mov_b32_e32 v1, s4
1065; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v1, v0
1066; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1067; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1068; GFX8-NEXT:    flat_store_dword v[0:1], v2
1069; GFX8-NEXT:    s_endpgm
1070;
1071; GFX9-NODL-LABEL: udot4_multiuse_mul1:
1072; GFX9-NODL:       ; %bb.0: ; %entry
1073; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1074; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1075; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
1076; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1077; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1078; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1079; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
1080; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1081; GFX9-NODL-NEXT:    s_and_b32 s5, s3, s2
1082; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1083; GFX9-NODL-NEXT:    s_bfe_u32 s7, s4, 0x80008
1084; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s2
1085; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s10
1086; GFX9-NODL-NEXT:    s_bfe_u32 s6, s3, 0x80008
1087; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s5, v0, v1
1088; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s7
1089; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
1090; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s6, v2, v1
1091; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
1092; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1093; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s9
1094; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
1095; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
1096; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
1097; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s4
1098; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s3, v1, v0
1099; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1100; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1101; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1102; GFX9-NODL-NEXT:    s_endpgm
1103;
1104; GFX9-DL-LABEL: udot4_multiuse_mul1:
1105; GFX9-DL:       ; %bb.0: ; %entry
1106; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1107; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1108; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
1109; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1110; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1111; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1112; GFX9-DL-NEXT:    s_load_dword s10, s[0:1], 0x0
1113; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1114; GFX9-DL-NEXT:    s_and_b32 s5, s3, s2
1115; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
1116; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x80008
1117; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
1118; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s10
1119; GFX9-DL-NEXT:    s_bfe_u32 s6, s3, 0x80008
1120; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s5, v0, v1
1121; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s7
1122; GFX9-DL-NEXT:    s_bfe_u32 s9, s4, 0x80010
1123; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s6, v2, v1
1124; GFX9-DL-NEXT:    s_bfe_u32 s8, s3, 0x80010
1125; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1126; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s9
1127; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 24
1128; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
1129; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 24
1130; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s4
1131; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s3, v1, v0
1132; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1133; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1134; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1135; GFX9-DL-NEXT:    s_endpgm
1136;
1137; GFX10-DL-LABEL: udot4_multiuse_mul1:
1138; GFX10-DL:       ; %bb.0: ; %entry
1139; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1140; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1141; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
1142; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1143; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1144; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1145; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1146; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1147; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1148; GFX10-DL-NEXT:    s_and_b32 s6, s3, s2
1149; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
1150; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
1151; GFX10-DL-NEXT:    s_bfe_u32 s5, s3, 0x80008
1152; GFX10-DL-NEXT:    s_bfe_u32 s7, s4, 0x80008
1153; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s2, v0
1154; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s5, s7, v0
1155; GFX10-DL-NEXT:    s_bfe_u32 s5, s3, 0x80010
1156; GFX10-DL-NEXT:    s_bfe_u32 s7, s4, 0x80010
1157; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s2, v0
1158; GFX10-DL-NEXT:    s_lshr_b32 s2, s3, 24
1159; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 24
1160; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s5, s7, v0
1161; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v0
1162; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1163; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1164; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1165; GFX10-DL-NEXT:    s_endpgm
1166                                               <4 x i8> addrspace(1)* %src2,
1167                                               i32 addrspace(1)* nocapture %dst) {
1168entry:
1169  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
1170  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
1171
1172  %v1e0 = extractelement <4 x i8> %vec1, i64 0
1173  %cv1e0 = zext i8 %v1e0 to i32
1174  %v2e0 = extractelement <4 x i8> %vec2, i64 0
1175  %cv2e0 = zext i8 %v2e0 to i32
1176  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
1177
1178  %v1e1 = extractelement <4 x i8> %vec1, i64 1
1179  %cv1e1 = zext i8 %v1e1 to i32
1180  %v2e1 = extractelement <4 x i8> %vec2, i64 1
1181  %cv2e1 = zext i8 %v2e1 to i32
1182  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
1183
1184  %v1e2 = extractelement <4 x i8> %vec1, i64 2
1185  %cv1e2 = zext i8 %v1e2 to i32
1186  %v2e2 = extractelement <4 x i8> %vec2, i64 2
1187  %cv2e2 = zext i8 %v2e2 to i32
1188  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
1189
1190  %v1e3 = extractelement <4 x i8> %vec1, i64 3
1191  %cv1e3 = zext i8 %v1e3 to i32
1192  %v2e3 = extractelement <4 x i8> %vec2, i64 3
1193  %cv2e3 = zext i8 %v2e3 to i32
1194  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
1195
1196  %acc = load i32, i32 addrspace(1)* %dst, align 4
1197  %add = add i32 %mul1, %acc
1198  %add1 = add i32 %mul2, %add
1199  %add2 = add i32 %add1, %mul1
1200  %add3 = add i32 %add2, %mul3
1201  %add4 = add i32 %add3, %mul4
1202
1203  store i32 %add4, i32 addrspace(1)* %dst, align 4
1204  ret void
1205}
1206
1207define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
1208; GFX7-LABEL: udot4_multiuse_add1:
1209; GFX7:       ; %bb.0: ; %entry
1210; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1211; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1212; GFX7-NEXT:    s_movk_i32 s8, 0xff
1213; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1214; GFX7-NEXT:    s_mov_b32 s2, -1
1215; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1216; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1217; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1218; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
1219; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1220; GFX7-NEXT:    s_and_b32 s6, s4, s8
1221; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
1222; GFX7-NEXT:    s_and_b32 s7, s5, s8
1223; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
1224; GFX7-NEXT:    v_mov_b32_e32 v0, s9
1225; GFX7-NEXT:    v_mov_b32_e32 v1, s12
1226; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v0, v1
1227; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
1228; GFX7-NEXT:    v_mov_b32_e32 v2, s7
1229; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
1230; GFX7-NEXT:    v_add_i32_e32 v1, vcc, s12, v0
1231; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v2, v0
1232; GFX7-NEXT:    v_mov_b32_e32 v2, s11
1233; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
1234; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v2, v0
1235; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
1236; GFX7-NEXT:    v_mov_b32_e32 v2, s5
1237; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v2, v0
1238; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
1239; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1240; GFX7-NEXT:    s_endpgm
1241;
1242; GFX8-LABEL: udot4_multiuse_add1:
1243; GFX8:       ; %bb.0: ; %entry
1244; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1245; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1246; GFX8-NEXT:    s_movk_i32 s2, 0xff
1247; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1248; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1249; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1250; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
1251; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1252; GFX8-NEXT:    s_and_b32 s5, s3, s2
1253; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x80008
1254; GFX8-NEXT:    s_and_b32 s2, s4, s2
1255; GFX8-NEXT:    s_bfe_u32 s6, s3, 0x80008
1256; GFX8-NEXT:    v_mov_b32_e32 v0, s7
1257; GFX8-NEXT:    v_mov_b32_e32 v1, s10
1258; GFX8-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
1259; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x80010
1260; GFX8-NEXT:    v_mov_b32_e32 v2, s2
1261; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x80010
1262; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s10, v0
1263; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v2, v0
1264; GFX8-NEXT:    v_mov_b32_e32 v2, s9
1265; GFX8-NEXT:    s_lshr_b32 s4, s4, 24
1266; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v2, v0
1267; GFX8-NEXT:    s_lshr_b32 s3, s3, 24
1268; GFX8-NEXT:    v_mov_b32_e32 v2, s4
1269; GFX8-NEXT:    v_mad_u32_u24 v0, s3, v2, v0
1270; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v1
1271; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1272; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1273; GFX8-NEXT:    flat_store_dword v[0:1], v2
1274; GFX8-NEXT:    s_endpgm
1275;
1276; GFX9-NODL-LABEL: udot4_multiuse_add1:
1277; GFX9-NODL:       ; %bb.0: ; %entry
1278; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1279; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1280; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
1281; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1282; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1283; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1284; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
1285; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1286; GFX9-NODL-NEXT:    s_and_b32 s5, s3, s2
1287; GFX9-NODL-NEXT:    s_bfe_u32 s7, s4, 0x80008
1288; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1289; GFX9-NODL-NEXT:    s_bfe_u32 s6, s3, 0x80008
1290; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s7
1291; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s10
1292; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
1293; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
1294; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s2
1295; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
1296; GFX9-NODL-NEXT:    v_add_u32_e32 v1, s10, v0
1297; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s5, v2, v0
1298; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s9
1299; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
1300; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s8, v2, v0
1301; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
1302; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
1303; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s3, v2, v0
1304; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v0, v1
1305; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1306; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1307; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1308; GFX9-NODL-NEXT:    s_endpgm
1309;
1310; GFX9-DL-LABEL: udot4_multiuse_add1:
1311; GFX9-DL:       ; %bb.0: ; %entry
1312; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1313; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1314; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
1315; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1316; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1317; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1318; GFX9-DL-NEXT:    s_load_dword s10, s[0:1], 0x0
1319; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1320; GFX9-DL-NEXT:    s_and_b32 s5, s3, s2
1321; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x80008
1322; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
1323; GFX9-DL-NEXT:    s_bfe_u32 s6, s3, 0x80008
1324; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s7
1325; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s10
1326; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
1327; GFX9-DL-NEXT:    s_bfe_u32 s9, s4, 0x80010
1328; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
1329; GFX9-DL-NEXT:    s_bfe_u32 s8, s3, 0x80010
1330; GFX9-DL-NEXT:    v_add_u32_e32 v1, s10, v0
1331; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s5, v2, v0
1332; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s9
1333; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 24
1334; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s8, v2, v0
1335; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 24
1336; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
1337; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s3, v2, v0
1338; GFX9-DL-NEXT:    v_add_u32_e32 v2, v0, v1
1339; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1340; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1341; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1342; GFX9-DL-NEXT:    s_endpgm
1343;
1344; GFX10-DL-LABEL: udot4_multiuse_add1:
1345; GFX10-DL:       ; %bb.0: ; %entry
1346; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1347; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1348; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
1349; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1350; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1351; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1352; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1353; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1354; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1355; GFX10-DL-NEXT:    s_bfe_u32 s6, s3, 0x80008
1356; GFX10-DL-NEXT:    s_bfe_u32 s7, s4, 0x80008
1357; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
1358; GFX10-DL-NEXT:    s_and_b32 s8, s3, s2
1359; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
1360; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s7, v0
1361; GFX10-DL-NEXT:    s_bfe_u32 s6, s3, 0x80010
1362; GFX10-DL-NEXT:    s_bfe_u32 s7, s4, 0x80010
1363; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s8, s2, v0
1364; GFX10-DL-NEXT:    s_lshr_b32 s2, s3, 24
1365; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 24
1366; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, s5, v0
1367; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s6, s7, v1
1368; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s2, s3, v1
1369; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v1, v0
1370; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1371; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1372; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1373; GFX10-DL-NEXT:    s_endpgm
1374                                               <4 x i8> addrspace(1)* %src2,
1375                                               i32 addrspace(1)* nocapture %dst) {
1376entry:
1377  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
1378  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
1379
1380  %v1e0 = extractelement <4 x i8> %vec1, i64 0
1381  %cv1e0 = zext i8 %v1e0 to i32
1382  %v2e0 = extractelement <4 x i8> %vec2, i64 0
1383  %cv2e0 = zext i8 %v2e0 to i32
1384  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
1385
1386  %v1e1 = extractelement <4 x i8> %vec1, i64 1
1387  %cv1e1 = zext i8 %v1e1 to i32
1388  %v2e1 = extractelement <4 x i8> %vec2, i64 1
1389  %cv2e1 = zext i8 %v2e1 to i32
1390  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
1391
1392  %v1e2 = extractelement <4 x i8> %vec1, i64 2
1393  %cv1e2 = zext i8 %v1e2 to i32
1394  %v2e2 = extractelement <4 x i8> %vec2, i64 2
1395  %cv2e2 = zext i8 %v2e2 to i32
1396  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
1397
1398  %v1e3 = extractelement <4 x i8> %vec1, i64 3
1399  %cv1e3 = zext i8 %v1e3 to i32
1400  %v2e3 = extractelement <4 x i8> %vec2, i64 3
1401  %cv2e3 = zext i8 %v2e3 to i32
1402  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
1403
1404  %acc = load i32, i32 addrspace(1)* %dst, align 4
1405  %add1 = add i32 %mul2, %acc
1406  %add = add i32 %add1, %acc
1407  %add2 = add i32 %add1, %mul1
1408  %add3 = add i32 %add2, %mul3
1409  %add4 = add i32 %add3, %mul4
1410  %res = add i32 %add4, %add
1411  store i32 %res, i32 addrspace(1)* %dst, align 4
1412  ret void
1413}
1414
1415define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1,
1416; GFX7-LABEL: notdot4_mixedtypes:
1417; GFX7:       ; %bb.0: ; %entry
1418; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1419; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1420; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1421; GFX7-NEXT:    s_mov_b32 s2, -1
1422; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1423; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1424; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
1425; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
1426; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1427; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1428; GFX7-NEXT:    s_sext_i32_i8 s7, s6
1429; GFX7-NEXT:    s_bfe_u32 s9, s6, 0x80008
1430; GFX7-NEXT:    s_sext_i32_i8 s5, s4
1431; GFX7-NEXT:    s_and_b32 s7, s7, s8
1432; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80008
1433; GFX7-NEXT:    v_mov_b32_e32 v1, s9
1434; GFX7-NEXT:    s_bfe_u32 s11, s6, 0x80010
1435; GFX7-NEXT:    s_and_b32 s5, s5, s8
1436; GFX7-NEXT:    v_mov_b32_e32 v3, s7
1437; GFX7-NEXT:    s_bfe_u32 s12, s4, 0x80010
1438; GFX7-NEXT:    s_lshr_b32 s6, s6, 24
1439; GFX7-NEXT:    v_mov_b32_e32 v2, s11
1440; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
1441; GFX7-NEXT:    s_waitcnt vmcnt(0)
1442; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
1443; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v3, v0
1444; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v2, v0
1445; GFX7-NEXT:    v_mov_b32_e32 v1, s6
1446; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1447; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
1448; GFX7-NEXT:    s_endpgm
1449;
1450; GFX8-LABEL: notdot4_mixedtypes:
1451; GFX8:       ; %bb.0: ; %entry
1452; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1453; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1454; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1455; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
1456; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1457; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1458; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
1459; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
1460; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1461; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x80008
1462; GFX8-NEXT:    s_sext_i32_i8 s3, s2
1463; GFX8-NEXT:    v_mov_b32_e32 v3, s1
1464; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80010
1465; GFX8-NEXT:    s_bfe_u32 s1, s0, 0x80008
1466; GFX8-NEXT:    v_mov_b32_e32 v4, s3
1467; GFX8-NEXT:    s_sext_i32_i8 s4, s0
1468; GFX8-NEXT:    s_bfe_u32 s3, s0, 0x80010
1469; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
1470; GFX8-NEXT:    v_mov_b32_e32 v5, s5
1471; GFX8-NEXT:    s_lshr_b32 s0, s0, 24
1472; GFX8-NEXT:    s_waitcnt vmcnt(0)
1473; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
1474; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v4, v2
1475; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v5, v2
1476; GFX8-NEXT:    v_mov_b32_e32 v3, s2
1477; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
1478; GFX8-NEXT:    flat_store_short v[0:1], v2
1479; GFX8-NEXT:    s_endpgm
1480;
1481; GFX9-NODL-LABEL: notdot4_mixedtypes:
1482; GFX9-NODL:       ; %bb.0: ; %entry
1483; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1484; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1485; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1486; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
1487; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1488; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1489; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
1490; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
1491; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1492; GFX9-NODL-NEXT:    s_bfe_u32 s1, s2, 0x80008
1493; GFX9-NODL-NEXT:    s_sext_i32_i8 s3, s2
1494; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
1495; GFX9-NODL-NEXT:    s_bfe_u32 s5, s2, 0x80010
1496; GFX9-NODL-NEXT:    s_bfe_u32 s1, s0, 0x80008
1497; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s3
1498; GFX9-NODL-NEXT:    s_sext_i32_i8 s4, s0
1499; GFX9-NODL-NEXT:    s_bfe_u32 s3, s0, 0x80010
1500; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
1501; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s5
1502; GFX9-NODL-NEXT:    s_lshr_b32 s0, s0, 24
1503; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
1504; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
1505; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s4, v4, v2
1506; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s3, v5, v2
1507; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
1508; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
1509; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
1510; GFX9-NODL-NEXT:    s_endpgm
1511;
1512; GFX9-DL-LABEL: notdot4_mixedtypes:
1513; GFX9-DL:       ; %bb.0: ; %entry
1514; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1515; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1516; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1517; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
1518; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1519; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1520; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
1521; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
1522; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1523; GFX9-DL-NEXT:    s_bfe_u32 s1, s2, 0x80008
1524; GFX9-DL-NEXT:    s_sext_i32_i8 s3, s2
1525; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
1526; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x80010
1527; GFX9-DL-NEXT:    s_bfe_u32 s1, s0, 0x80008
1528; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s3
1529; GFX9-DL-NEXT:    s_sext_i32_i8 s4, s0
1530; GFX9-DL-NEXT:    s_bfe_u32 s3, s0, 0x80010
1531; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 24
1532; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s5
1533; GFX9-DL-NEXT:    s_lshr_b32 s0, s0, 24
1534; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
1535; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
1536; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s4, v4, v2
1537; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s3, v5, v2
1538; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
1539; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
1540; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
1541; GFX9-DL-NEXT:    s_endpgm
1542;
1543; GFX10-DL-LABEL: notdot4_mixedtypes:
1544; GFX10-DL:       ; %bb.0: ; %entry
1545; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1546; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1547; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1548; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1549; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1550; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1551; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
1552; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
1553; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
1554; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1555; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x80008
1556; GFX10-DL-NEXT:    s_bfe_u32 s3, s1, 0x80008
1557; GFX10-DL-NEXT:    s_sext_i32_i8 s4, s0
1558; GFX10-DL-NEXT:    s_sext_i32_i8 s5, s1
1559; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
1560; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v2
1561; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x80010
1562; GFX10-DL-NEXT:    s_bfe_u32 s3, s1, 0x80010
1563; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 24
1564; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 24
1565; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s4, s5, v2
1566; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v2
1567; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s1, v2
1568; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
1569; GFX10-DL-NEXT:    s_endpgm
1570                                              <4 x i8> addrspace(1)* %src2,
1571                                              i16 addrspace(1)* nocapture %dst) {
1572entry:
1573  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
1574  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
1575
1576  %v1e0 = extractelement <4 x i8> %vec1, i64 0
1577  %cv1e0 = sext i8 %v1e0 to i16
1578  %v2e0 = extractelement <4 x i8> %vec2, i64 0
1579  %cv2e0 = sext i8 %v2e0 to i16
1580  %mul1 = mul nuw nsw i16 %cv1e0, %cv2e0
1581
1582  %v1e1 = extractelement <4 x i8> %vec1, i64 1
1583  %cv1e1 = zext i8 %v1e1 to i16
1584  %v2e1 = extractelement <4 x i8> %vec2, i64 1
1585  %cv2e1 = zext i8 %v2e1 to i16
1586  %mul2 = mul nuw nsw i16 %cv1e1, %cv2e1
1587
1588  %v1e2 = extractelement <4 x i8> %vec1, i64 2
1589  %cv1e2 = zext i8 %v1e2 to i16
1590  %v2e2 = extractelement <4 x i8> %vec2, i64 2
1591  %cv2e2 = zext i8 %v2e2 to i16
1592  %mul3 = mul nuw nsw i16 %cv1e2, %cv2e2
1593
1594  %v1e3 = extractelement <4 x i8> %vec1, i64 3
1595  %cv1e3 = zext i8 %v1e3 to i16
1596  %v2e3 = extractelement <4 x i8> %vec2, i64 3
1597  %cv2e3 = zext i8 %v2e3 to i16
1598  %mul4 = mul nuw nsw i16 %cv1e3, %cv2e3
1599
1600  %acc = load i16, i16 addrspace(1)* %dst, align 2
1601  %add1 = add i16 %mul2, %acc
1602  %add2 = add i16 %add1, %mul1
1603  %add3 = add i16 %add2, %mul3
1604  %add4 = add i16 %add3, %mul4
1605
1606  store i16 %add4, i16 addrspace(1)* %dst, align 2
1607  ret void
1608}
1609
1610; TODO: cleanup s_lshr_b32 and support this pattern.
1611define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
1612; GFX7-LABEL: udot4_acc32_vecMul:
1613; GFX7:       ; %bb.0: ; %entry
1614; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1615; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1616; GFX7-NEXT:    s_movk_i32 s11, 0xff
1617; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1618; GFX7-NEXT:    s_mov_b32 s2, -1
1619; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1620; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1621; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1622; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1623; GFX7-NEXT:    s_lshr_b32 s6, s4, 24
1624; GFX7-NEXT:    s_bfe_u32 s7, s4, 0x80008
1625; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
1626; GFX7-NEXT:    s_lshr_b32 s8, s5, 24
1627; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
1628; GFX7-NEXT:    s_bfe_u32 s12, s5, 0x80010
1629; GFX7-NEXT:    s_and_b32 s5, s5, s11
1630; GFX7-NEXT:    s_and_b32 s4, s4, s11
1631; GFX7-NEXT:    s_load_dword s11, s[0:1], 0x0
1632; GFX7-NEXT:    v_mov_b32_e32 v0, s5
1633; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1634; GFX7-NEXT:    v_mov_b32_e32 v1, s11
1635; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v0, v1
1636; GFX7-NEXT:    v_mov_b32_e32 v1, s9
1637; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
1638; GFX7-NEXT:    v_mov_b32_e32 v1, s12
1639; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
1640; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1641; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
1642; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1643; GFX7-NEXT:    s_endpgm
1644;
1645; GFX8-LABEL: udot4_acc32_vecMul:
1646; GFX8:       ; %bb.0: ; %entry
1647; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1648; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1649; GFX8-NEXT:    s_movk_i32 s2, 0xff
1650; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1651; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1652; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1653; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1654; GFX8-NEXT:    s_lshr_b32 s5, s3, 24
1655; GFX8-NEXT:    s_lshr_b32 s6, s4, 24
1656; GFX8-NEXT:    s_bfe_u32 s7, s3, 0x80010
1657; GFX8-NEXT:    v_lshrrev_b16_e64 v0, 8, s3
1658; GFX8-NEXT:    s_and_b32 s3, s3, s2
1659; GFX8-NEXT:    s_and_b32 s2, s4, s2
1660; GFX8-NEXT:    s_bfe_u32 s8, s4, 0x80010
1661; GFX8-NEXT:    v_lshrrev_b16_e64 v1, 8, s4
1662; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
1663; GFX8-NEXT:    v_mov_b32_e32 v2, s2
1664; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1665; GFX8-NEXT:    v_mov_b32_e32 v3, s4
1666; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v2, v3
1667; GFX8-NEXT:    v_mad_u32_u24 v0, v0, v1, v2
1668; GFX8-NEXT:    v_mov_b32_e32 v1, s8
1669; GFX8-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
1670; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1671; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v1, v0
1672; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1673; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1674; GFX8-NEXT:    flat_store_dword v[0:1], v2
1675; GFX8-NEXT:    s_endpgm
1676;
1677; GFX9-NODL-LABEL: udot4_acc32_vecMul:
1678; GFX9-NODL:       ; %bb.0: ; %entry
1679; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1680; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1681; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
1682; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1683; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1684; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1685; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1686; GFX9-NODL-NEXT:    s_lshr_b32 s5, s3, 24
1687; GFX9-NODL-NEXT:    s_lshr_b32 s6, s4, 24
1688; GFX9-NODL-NEXT:    s_bfe_u32 s7, s3, 0x80010
1689; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v0, 8, s3
1690; GFX9-NODL-NEXT:    s_and_b32 s3, s3, s2
1691; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1692; GFX9-NODL-NEXT:    s_bfe_u32 s8, s4, 0x80010
1693; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v1, 8, s4
1694; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
1695; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s2
1696; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1697; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s4
1698; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s3, v2, v3
1699; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, v0, v1, v2
1700; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s8
1701; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
1702; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1703; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s5, v1, v0
1704; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1705; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1706; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1707; GFX9-NODL-NEXT:    s_endpgm
1708;
1709; GFX9-DL-LABEL: udot4_acc32_vecMul:
1710; GFX9-DL:       ; %bb.0: ; %entry
1711; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1712; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1713; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
1714; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1715; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1716; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1717; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1718; GFX9-DL-NEXT:    s_lshr_b32 s5, s3, 24
1719; GFX9-DL-NEXT:    s_lshr_b32 s6, s4, 24
1720; GFX9-DL-NEXT:    s_bfe_u32 s7, s3, 0x80010
1721; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v0, 8, s3
1722; GFX9-DL-NEXT:    s_and_b32 s3, s3, s2
1723; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
1724; GFX9-DL-NEXT:    s_bfe_u32 s8, s4, 0x80010
1725; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, s4
1726; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
1727; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
1728; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1729; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
1730; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s3, v2, v3
1731; GFX9-DL-NEXT:    v_mad_u32_u24 v0, v0, v1, v2
1732; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s8
1733; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
1734; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
1735; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v1, v0
1736; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1737; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1738; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1739; GFX9-DL-NEXT:    s_endpgm
1740;
1741; GFX10-DL-LABEL: udot4_acc32_vecMul:
1742; GFX10-DL:       ; %bb.0: ; %entry
1743; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1744; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1745; GFX10-DL-NEXT:    s_movk_i32 s3, 0xff
1746; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
1747; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1748; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1749; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
1750; GFX10-DL-NEXT:    s_load_dword s5, s[6:7], 0x0
1751; GFX10-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
1752; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1753; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
1754; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
1755; GFX10-DL-NEXT:    s_and_b32 s7, s4, s3
1756; GFX10-DL-NEXT:    s_and_b32 s3, s5, s3
1757; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s6
1758; GFX10-DL-NEXT:    v_and_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
1759; GFX10-DL-NEXT:    v_and_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
1760; GFX10-DL-NEXT:    s_bfe_u32 s2, s4, 0x80010
1761; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 24
1762; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s7, s3, v2
1763; GFX10-DL-NEXT:    s_bfe_u32 s3, s5, 0x80010
1764; GFX10-DL-NEXT:    s_lshr_b32 s5, s5, 24
1765; GFX10-DL-NEXT:    v_mad_u32_u24 v0, v0, v1, v2
1766; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s2, s3, v0
1767; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s4, s5, v0
1768; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1769; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1770; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1771; GFX10-DL-NEXT:    s_endpgm
1772                                              <4 x i8> addrspace(1)* %src2,
1773                                              i32 addrspace(1)* nocapture %dst) {
1774entry:
1775  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
1776  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
1777
1778  %cvec1 = zext <4 x i8> %vec1 to <4 x i32>
1779  %cvec2 = zext <4 x i8> %vec2 to <4 x i32>
1780
1781  %mul = mul <4 x i32> %cvec1, %cvec2
1782  %mul0 = extractelement <4 x i32> %mul, i64 0
1783  %mul1 = extractelement <4 x i32> %mul, i64 1
1784  %mul2 = extractelement <4 x i32> %mul, i64 2
1785  %mul3 = extractelement <4 x i32> %mul, i64 3
1786
1787  %acc = load i32, i32 addrspace(1)* %dst, align 4
1788  %add1 = add i32 %mul0, %acc
1789  %add2 = add i32 %add1, %mul1
1790  %add3 = add i32 %add2, %mul2
1791  %add4 = add i32 %add3, %mul3
1792
1793  store i32 %add4, i32 addrspace(1)* %dst, align 4
1794  ret void
1795}
1796
1797; TODO: This pattern should be recognized.
1798define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
1799; GFX7-LABEL: udot4_acc16_vecMul:
1800; GFX7:       ; %bb.0: ; %entry
1801; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1802; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1803; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1804; GFX7-NEXT:    s_mov_b32 s2, -1
1805; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1806; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
1807; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
1808; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1809; GFX7-NEXT:    s_movk_i32 s7, 0xff
1810; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1811; GFX7-NEXT:    s_bfe_u32 s10, s6, 0x80008
1812; GFX7-NEXT:    s_bfe_u32 s12, s6, 0x80010
1813; GFX7-NEXT:    s_lshr_b32 s9, s6, 24
1814; GFX7-NEXT:    s_and_b32 s6, s6, s7
1815; GFX7-NEXT:    s_lshr_b32 s5, s4, 24
1816; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
1817; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
1818; GFX7-NEXT:    s_and_b32 s4, s4, s7
1819; GFX7-NEXT:    v_mov_b32_e32 v1, s6
1820; GFX7-NEXT:    v_mov_b32_e32 v2, s10
1821; GFX7-NEXT:    v_mov_b32_e32 v3, s12
1822; GFX7-NEXT:    s_waitcnt vmcnt(0)
1823; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1824; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v2, v0
1825; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
1826; GFX7-NEXT:    v_mov_b32_e32 v1, s9
1827; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1828; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
1829; GFX7-NEXT:    s_endpgm
1830;
1831; GFX8-LABEL: udot4_acc16_vecMul:
1832; GFX8:       ; %bb.0: ; %entry
1833; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1834; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1835; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1836; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1837; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1838; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
1839; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
1840; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
1841; GFX8-NEXT:    s_movk_i32 s0, 0xff
1842; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1843; GFX8-NEXT:    s_and_b32 s6, s1, s0
1844; GFX8-NEXT:    s_and_b32 s0, s2, s0
1845; GFX8-NEXT:    v_mov_b32_e32 v5, s0
1846; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
1847; GFX8-NEXT:    v_lshrrev_b16_e64 v3, 8, s2
1848; GFX8-NEXT:    v_lshrrev_b16_e64 v4, 8, s1
1849; GFX8-NEXT:    s_lshr_b32 s4, s2, 24
1850; GFX8-NEXT:    s_bfe_u32 s5, s1, 0x80010
1851; GFX8-NEXT:    v_mov_b32_e32 v6, s7
1852; GFX8-NEXT:    s_lshr_b32 s3, s1, 24
1853; GFX8-NEXT:    s_waitcnt vmcnt(0)
1854; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
1855; GFX8-NEXT:    v_mad_u32_u24 v2, v4, v3, v2
1856; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v6, v2
1857; GFX8-NEXT:    v_mov_b32_e32 v3, s4
1858; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
1859; GFX8-NEXT:    flat_store_short v[0:1], v2
1860; GFX8-NEXT:    s_endpgm
1861;
1862; GFX9-NODL-LABEL: udot4_acc16_vecMul:
1863; GFX9-NODL:       ; %bb.0: ; %entry
1864; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1865; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1866; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0xffff
1867; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1868; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
1869; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
1870; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1871; GFX9-NODL-NEXT:    s_lshr_b32 s5, s2, 16
1872; GFX9-NODL-NEXT:    s_lshr_b32 s7, s3, 16
1873; GFX9-NODL-NEXT:    s_lshr_b32 s4, s2, 24
1874; GFX9-NODL-NEXT:    v_and_b32_sdwa v4, v0, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1875; GFX9-NODL-NEXT:    s_lshr_b32 s6, s3, 24
1876; GFX9-NODL-NEXT:    v_and_b32_sdwa v3, v0, s7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1877; GFX9-NODL-NEXT:    v_lshl_or_b32 v3, s6, 16, v3
1878; GFX9-NODL-NEXT:    v_lshl_or_b32 v4, s4, 16, v4
1879; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v3, v4, v3
1880; GFX9-NODL-NEXT:    v_and_b32_sdwa v4, v0, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1881; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v2, 8, s3
1882; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v1, 8, s2
1883; GFX9-NODL-NEXT:    v_and_b32_sdwa v0, v0, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1884; GFX9-NODL-NEXT:    v_lshl_or_b32 v2, v2, 16, v4
1885; GFX9-NODL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
1886; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v2, v0, v2
1887; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1888; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1889; GFX9-NODL-NEXT:    global_load_ushort v4, v[0:1], off
1890; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
1891; GFX9-NODL-NEXT:    v_add_u32_e32 v4, v2, v4
1892; GFX9-NODL-NEXT:    v_add_u32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1893; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v2, v3
1894; GFX9-NODL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1895; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
1896; GFX9-NODL-NEXT:    s_endpgm
1897;
1898; GFX9-DL-LABEL: udot4_acc16_vecMul:
1899; GFX9-DL:       ; %bb.0: ; %entry
1900; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1901; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1902; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0xffff
1903; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1904; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1905; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1906; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1907; GFX9-DL-NEXT:    s_lshr_b32 s5, s2, 16
1908; GFX9-DL-NEXT:    s_lshr_b32 s7, s3, 16
1909; GFX9-DL-NEXT:    s_lshr_b32 s4, s2, 24
1910; GFX9-DL-NEXT:    v_and_b32_sdwa v4, v0, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1911; GFX9-DL-NEXT:    s_lshr_b32 s6, s3, 24
1912; GFX9-DL-NEXT:    v_and_b32_sdwa v3, v0, s7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1913; GFX9-DL-NEXT:    v_lshl_or_b32 v3, s6, 16, v3
1914; GFX9-DL-NEXT:    v_lshl_or_b32 v4, s4, 16, v4
1915; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, v4, v3
1916; GFX9-DL-NEXT:    v_and_b32_sdwa v4, v0, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1917; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v2, 8, s3
1918; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, s2
1919; GFX9-DL-NEXT:    v_and_b32_sdwa v0, v0, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1920; GFX9-DL-NEXT:    v_lshl_or_b32 v2, v2, 16, v4
1921; GFX9-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
1922; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v2, v0, v2
1923; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1924; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1925; GFX9-DL-NEXT:    global_load_ushort v4, v[0:1], off
1926; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
1927; GFX9-DL-NEXT:    v_add_u32_e32 v4, v2, v4
1928; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1929; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v3
1930; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1931; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
1932; GFX9-DL-NEXT:    s_endpgm
1933;
1934; GFX10-DL-LABEL: udot4_acc16_vecMul:
1935; GFX10-DL:       ; %bb.0: ; %entry
1936; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1937; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1938; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
1939; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1940; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1941; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1942; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1943; GFX10-DL-NEXT:    global_load_ushort v3, v[0:1], off
1944; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
1945; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
1946; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1947; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v4, 8, s0
1948; GFX10-DL-NEXT:    v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1949; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v5, 8, s1
1950; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v2, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1951; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 16
1952; GFX10-DL-NEXT:    s_lshr_b32 s3, s1, 16
1953; GFX10-DL-NEXT:    v_lshl_or_b32 v4, v4, 16, v7
1954; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 24
1955; GFX10-DL-NEXT:    v_lshl_or_b32 v5, v5, 16, v6
1956; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 24
1957; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v2, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1958; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v2, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1959; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v5
1960; GFX10-DL-NEXT:    v_lshl_or_b32 v5, s1, 16, v6
1961; GFX10-DL-NEXT:    v_lshl_or_b32 v2, s0, 16, v2
1962; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v5
1963; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
1964; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v4, v3
1965; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1966; GFX10-DL-NEXT:    v_add_nc_u32_e32 v3, v3, v2
1967; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1968; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
1969; GFX10-DL-NEXT:    s_endpgm
1970                                              <4 x i8> addrspace(1)* %src2,
1971                                              i16 addrspace(1)* nocapture %dst) {
1972entry:
1973  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
1974  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
1975
1976  %cvec1 = zext <4 x i8> %vec1 to <4 x i16>
1977  %cvec2 = zext <4 x i8> %vec2 to <4 x i16>
1978
1979  %mul = mul <4 x i16> %cvec1, %cvec2
1980  %mul0 = extractelement <4 x i16> %mul, i64 0
1981  %mul1 = extractelement <4 x i16> %mul, i64 1
1982  %mul2 = extractelement <4 x i16> %mul, i64 2
1983  %mul3 = extractelement <4 x i16> %mul, i64 3
1984
1985  %acc = load i16, i16 addrspace(1)* %dst, align 4
1986  %add1 = add i16 %mul0, %acc
1987  %add2 = add i16 %add1, %mul1
1988  %add3 = add i16 %add2, %mul2
1989  %add4 = add i16 %add3, %mul3
1990
1991  store i16 %add4, i16 addrspace(1)* %dst, align 4
1992  ret void
1993}
1994
1995; TODO: Support this pattern.
1996define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
1997; GFX7-LABEL: udot4_acc8_vecMul:
1998; GFX7:       ; %bb.0: ; %entry
1999; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2000; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2001; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2002; GFX7-NEXT:    s_mov_b32 s2, -1
2003; GFX7-NEXT:    s_movk_i32 s8, 0xff
2004; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2005; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
2006; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2007; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2008; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2009; GFX7-NEXT:    s_bfe_u32 s6, s4, 0x80008
2010; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80008
2011; GFX7-NEXT:    s_lshr_b32 s11, s5, 16
2012; GFX7-NEXT:    s_lshr_b32 s12, s5, 24
2013; GFX7-NEXT:    v_mov_b32_e32 v3, s10
2014; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
2015; GFX7-NEXT:    v_mov_b32_e32 v2, s11
2016; GFX7-NEXT:    s_lshr_b32 s9, s4, 24
2017; GFX7-NEXT:    v_mov_b32_e32 v1, s12
2018; GFX7-NEXT:    s_mul_i32 s4, s4, s5
2019; GFX7-NEXT:    v_mul_u32_u24_e32 v1, s9, v1
2020; GFX7-NEXT:    v_mul_u32_u24_e32 v2, s7, v2
2021; GFX7-NEXT:    v_mul_u32_u24_e32 v3, s6, v3
2022; GFX7-NEXT:    s_and_b32 s5, s4, s8
2023; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
2024; GFX7-NEXT:    v_and_b32_e32 v2, s8, v2
2025; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
2026; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
2027; GFX7-NEXT:    v_or_b32_e32 v2, s5, v3
2028; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
2029; GFX7-NEXT:    v_and_b32_e32 v2, 0xffff, v2
2030; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
2031; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 8, v1
2032; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
2033; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
2034; GFX7-NEXT:    s_waitcnt vmcnt(0)
2035; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
2036; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
2037; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
2038; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
2039; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
2040; GFX7-NEXT:    s_endpgm
2041;
2042; GFX8-LABEL: udot4_acc8_vecMul:
2043; GFX8:       ; %bb.0: ; %entry
2044; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2045; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2046; GFX8-NEXT:    s_movk_i32 s8, 0xff
2047; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2048; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2049; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2050; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
2051; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
2052; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
2053; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2054; GFX8-NEXT:    v_mov_b32_e32 v3, s0
2055; GFX8-NEXT:    v_mov_b32_e32 v4, s1
2056; GFX8-NEXT:    s_and_b32 s7, s1, s8
2057; GFX8-NEXT:    s_lshr_b32 s2, s0, 24
2058; GFX8-NEXT:    s_lshr_b32 s3, s1, 24
2059; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
2060; GFX8-NEXT:    v_mul_u32_u24_sdwa v3, v3, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
2061; GFX8-NEXT:    s_and_b32 s5, s0, s8
2062; GFX8-NEXT:    v_mov_b32_e32 v4, s7
2063; GFX8-NEXT:    v_mul_u32_u24_e32 v4, s5, v4
2064; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x80010
2065; GFX8-NEXT:    v_mov_b32_e32 v5, s6
2066; GFX8-NEXT:    v_mov_b32_e32 v6, s3
2067; GFX8-NEXT:    v_mov_b32_e32 v7, s2
2068; GFX8-NEXT:    v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2069; GFX8-NEXT:    v_mul_u32_u24_e32 v5, s4, v5
2070; GFX8-NEXT:    v_mul_u32_u24_sdwa v6, v7, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2071; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff, v3
2072; GFX8-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2073; GFX8-NEXT:    v_or_b32_e32 v4, v3, v5
2074; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
2075; GFX8-NEXT:    s_waitcnt vmcnt(0)
2076; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
2077; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
2078; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2079; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2080; GFX8-NEXT:    flat_store_byte v[0:1], v2
2081; GFX8-NEXT:    s_endpgm
2082;
2083; GFX9-NODL-LABEL: udot4_acc8_vecMul:
2084; GFX9-NODL:       ; %bb.0: ; %entry
2085; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2086; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2087; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2088; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
2089; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2090; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2091; GFX9-NODL-NEXT:    global_load_ubyte v2, v[0:1], off
2092; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
2093; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2094; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
2095; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s2
2096; GFX9-NODL-NEXT:    s_lshr_b32 s1, s2, 16
2097; GFX9-NODL-NEXT:    s_lshr_b32 s3, s2, 24
2098; GFX9-NODL-NEXT:    v_mul_lo_u16_e32 v3, s0, v3
2099; GFX9-NODL-NEXT:    v_mul_lo_u16_sdwa v4, s0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
2100; GFX9-NODL-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2101; GFX9-NODL-NEXT:    s_lshr_b32 s2, s0, 16
2102; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s1
2103; GFX9-NODL-NEXT:    s_lshr_b32 s4, s0, 24
2104; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s3
2105; GFX9-NODL-NEXT:    v_mul_lo_u16_sdwa v4, s4, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2106; GFX9-NODL-NEXT:    v_mul_lo_u16_e32 v5, s2, v5
2107; GFX9-NODL-NEXT:    v_and_b32_e32 v3, 0xffff, v3
2108; GFX9-NODL-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2109; GFX9-NODL-NEXT:    v_or_b32_e32 v4, v3, v4
2110; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
2111; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
2112; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v3, v2
2113; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v2, v5
2114; GFX9-NODL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2115; GFX9-NODL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2116; GFX9-NODL-NEXT:    global_store_byte v[0:1], v2, off
2117; GFX9-NODL-NEXT:    s_endpgm
2118;
2119; GFX9-DL-LABEL: udot4_acc8_vecMul:
2120; GFX9-DL:       ; %bb.0: ; %entry
2121; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2122; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2123; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2124; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
2125; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2126; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2127; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
2128; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
2129; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2130; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
2131; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s2
2132; GFX9-DL-NEXT:    s_lshr_b32 s1, s2, 16
2133; GFX9-DL-NEXT:    s_lshr_b32 s3, s2, 24
2134; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v3, s0, v3
2135; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, s0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
2136; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2137; GFX9-DL-NEXT:    s_lshr_b32 s2, s0, 16
2138; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s1
2139; GFX9-DL-NEXT:    s_lshr_b32 s4, s0, 24
2140; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s3
2141; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, s4, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2142; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v5, s2, v5
2143; GFX9-DL-NEXT:    v_and_b32_e32 v3, 0xffff, v3
2144; GFX9-DL-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2145; GFX9-DL-NEXT:    v_or_b32_e32 v4, v3, v4
2146; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
2147; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2148; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
2149; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
2150; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2151; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2152; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
2153; GFX9-DL-NEXT:    s_endpgm
2154;
2155; GFX10-DL-LABEL: udot4_acc8_vecMul:
2156; GFX10-DL:       ; %bb.0: ; %entry
2157; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2158; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2159; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2160; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2161; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2162; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2163; GFX10-DL-NEXT:    global_load_ubyte v2, v[0:1], off
2164; GFX10-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
2165; GFX10-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
2166; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2167; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, s0
2168; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v4, 8, s1
2169; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 24
2170; GFX10-DL-NEXT:    s_lshr_b32 s3, s1, 24
2171; GFX10-DL-NEXT:    s_lshr_b32 s4, s0, 16
2172; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v3, v3, v4
2173; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, s0, s1
2174; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s2, s3
2175; GFX10-DL-NEXT:    s_lshr_b32 s0, s1, 16
2176; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v3
2177; GFX10-DL-NEXT:    v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2178; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v4, 8, v5
2179; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, s4, s0
2180; GFX10-DL-NEXT:    v_and_b32_e32 v3, 0xffff, v3
2181; GFX10-DL-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2182; GFX10-DL-NEXT:    v_or_b32_e32 v4, v3, v4
2183; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
2184; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
2185; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
2186; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v2, v5
2187; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2188; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2189; GFX10-DL-NEXT:    global_store_byte v[0:1], v2, off
2190; GFX10-DL-NEXT:    s_endpgm
2191                                             <4 x i8> addrspace(1)* %src2,
2192                                             i8 addrspace(1)* nocapture %dst) {
2193entry:
2194  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
2195  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
2196
2197  %mul = mul <4 x i8> %vec1, %vec2
2198  %mul0 = extractelement <4 x i8> %mul, i64 0
2199  %mul1 = extractelement <4 x i8> %mul, i64 1
2200  %mul2 = extractelement <4 x i8> %mul, i64 2
2201  %mul3 = extractelement <4 x i8> %mul, i64 3
2202
2203  %acc = load i8, i8 addrspace(1)* %dst, align 4
2204  %add1 = add i8 %mul0, %acc
2205  %add2 = add i8 %add1, %mul1
2206  %add3 = add i8 %add2, %mul2
2207  %add4 = add i8 %add3, %mul3
2208
2209  store i8 %add4, i8 addrspace(1)* %dst, align 4
2210  ret void
2211}
2212