1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-NODL %s
5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
8
9define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
10; GFX7-LABEL: udot4_acc32:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
13; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
14; GFX7-NEXT:    s_movk_i32 s8, 0xff
15; GFX7-NEXT:    s_mov_b32 s3, 0xf000
16; GFX7-NEXT:    s_mov_b32 s2, -1
17; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
18; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
19; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
20; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
21; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
22; GFX7-NEXT:    s_and_b32 s6, s4, s8
23; GFX7-NEXT:    s_and_b32 s7, s5, s8
24; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
25; GFX7-NEXT:    v_mov_b32_e32 v0, s7
26; GFX7-NEXT:    v_mov_b32_e32 v1, s12
27; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
28; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
29; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
30; GFX7-NEXT:    v_mov_b32_e32 v1, s9
31; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
32; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
33; GFX7-NEXT:    v_mov_b32_e32 v1, s11
34; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
35; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
36; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
37; GFX7-NEXT:    v_mov_b32_e32 v1, s5
38; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
39; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
40; GFX7-NEXT:    s_endpgm
41;
42; GFX8-LABEL: udot4_acc32:
43; GFX8:       ; %bb.0: ; %entry
44; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
45; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
46; GFX8-NEXT:    s_movk_i32 s2, 0xff
47; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
48; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
49; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
50; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
51; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
52; GFX8-NEXT:    s_and_b32 s5, s3, s2
53; GFX8-NEXT:    s_and_b32 s2, s4, s2
54; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x80008
55; GFX8-NEXT:    v_mov_b32_e32 v0, s2
56; GFX8-NEXT:    v_mov_b32_e32 v1, s10
57; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x80010
58; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
59; GFX8-NEXT:    s_bfe_u32 s6, s3, 0x80008
60; GFX8-NEXT:    v_mov_b32_e32 v1, s7
61; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x80010
62; GFX8-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
63; GFX8-NEXT:    v_mov_b32_e32 v1, s9
64; GFX8-NEXT:    s_lshr_b32 s4, s4, 24
65; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
66; GFX8-NEXT:    s_lshr_b32 s3, s3, 24
67; GFX8-NEXT:    v_mov_b32_e32 v1, s4
68; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v1, v0
69; GFX8-NEXT:    v_mov_b32_e32 v0, s0
70; GFX8-NEXT:    v_mov_b32_e32 v1, s1
71; GFX8-NEXT:    flat_store_dword v[0:1], v2
72; GFX8-NEXT:    s_endpgm
73;
74; GFX9-NODL-LABEL: udot4_acc32:
75; GFX9-NODL:       ; %bb.0: ; %entry
76; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
77; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
78; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
79; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
80; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
81; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
82; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
83; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
84; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
85; GFX9-NODL-NEXT:    s_and_b32 s5, s3, s2
86; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
87; GFX9-NODL-NEXT:    s_bfe_u32 s7, s4, 0x80008
88; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
89; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s10
90; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
91; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s5, v1, v2
92; GFX9-NODL-NEXT:    s_bfe_u32 s6, s3, 0x80008
93; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s7
94; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
95; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s6, v2, v1
96; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s9
97; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
98; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
99; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
100; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
101; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s3, v2, v1
102; GFX9-NODL-NEXT:    global_store_dword v0, v1, s[0:1]
103; GFX9-NODL-NEXT:    s_endpgm
104;
105; GFX9-DL-LABEL: udot4_acc32:
106; GFX9-DL:       ; %bb.0: ; %entry
107; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
108; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
109; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
110; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
111; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
112; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
113; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
114; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
115; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
116; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
117; GFX9-DL-NEXT:    v_dot4_u32_u8 v1, s4, v1, v2
118; GFX9-DL-NEXT:    global_store_dword v0, v1, s[0:1]
119; GFX9-DL-NEXT:    s_endpgm
120;
121; GFX10-DL-LABEL: udot4_acc32:
122; GFX10-DL:       ; %bb.0: ; %entry
123; GFX10-DL-NEXT:    s_clause 0x1
124; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
125; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
126; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
127; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
128; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
129; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
130; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
131; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
132; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
133; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
134; GFX10-DL-NEXT:    v_dot4_u32_u8 v0, s0, s1, v0
135; GFX10-DL-NEXT:    global_store_dword v1, v0, s[4:5]
136; GFX10-DL-NEXT:    s_endpgm
137                                       <4 x i8> addrspace(1)* %src2,
138                                       i32 addrspace(1)* nocapture %dst) {
139entry:
140  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
141  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
142
143  %v1e0 = extractelement <4 x i8> %vec1, i64 0
144  %cv1e0 = zext i8 %v1e0 to i32
145  %v2e0 = extractelement <4 x i8> %vec2, i64 0
146  %cv2e0 = zext i8 %v2e0 to i32
147  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
148
149  %v1e1 = extractelement <4 x i8> %vec1, i64 1
150  %cv1e1 = zext i8 %v1e1 to i32
151  %v2e1 = extractelement <4 x i8> %vec2, i64 1
152  %cv2e1 = zext i8 %v2e1 to i32
153  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
154
155  %v1e2 = extractelement <4 x i8> %vec1, i64 2
156  %cv1e2 = zext i8 %v1e2 to i32
157  %v2e2 = extractelement <4 x i8> %vec2, i64 2
158  %cv2e2 = zext i8 %v2e2 to i32
159  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
160
161  %v1e3 = extractelement <4 x i8> %vec1, i64 3
162  %cv1e3 = zext i8 %v1e3 to i32
163  %v2e3 = extractelement <4 x i8> %vec2, i64 3
164  %cv2e3 = zext i8 %v2e3 to i32
165  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
166
167  %acc = load i32, i32 addrspace(1)* %dst, align 4
168  %mad1 = add i32 %mul1, %acc
169  %mad2 = add i32 %mad1, %mul2
170  %mad3 = add i32 %mad2, %mul3
171  %mad4 = add i32 %mad3, %mul4
172
173  store i32 %mad4, i32 addrspace(1)* %dst, align 4
174  ret void
175}
176
177define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1,
178; GFX7-LABEL: udot4_acc16:
179; GFX7:       ; %bb.0: ; %entry
180; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
181; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
182; GFX7-NEXT:    s_mov_b32 s3, 0xf000
183; GFX7-NEXT:    s_mov_b32 s2, -1
184; GFX7-NEXT:    s_movk_i32 s8, 0xff
185; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
186; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
187; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
188; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
189; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
190; GFX7-NEXT:    s_and_b32 s7, s4, s8
191; GFX7-NEXT:    s_and_b32 s6, s5, s8
192; GFX7-NEXT:    s_bfe_u32 s8, s5, 0x80008
193; GFX7-NEXT:    v_mov_b32_e32 v1, s6
194; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80010
195; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
196; GFX7-NEXT:    v_mov_b32_e32 v2, s8
197; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
198; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
199; GFX7-NEXT:    v_mov_b32_e32 v3, s10
200; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
201; GFX7-NEXT:    s_waitcnt vmcnt(0)
202; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
203; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
204; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
205; GFX7-NEXT:    v_mov_b32_e32 v1, s5
206; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
207; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
208; GFX7-NEXT:    s_endpgm
209;
210; GFX8-LABEL: udot4_acc16:
211; GFX8:       ; %bb.0: ; %entry
212; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
213; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
214; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
215; GFX8-NEXT:    v_mov_b32_e32 v0, s0
216; GFX8-NEXT:    v_mov_b32_e32 v1, s1
217; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
218; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
219; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
220; GFX8-NEXT:    s_movk_i32 s0, 0xff
221; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
222; GFX8-NEXT:    s_and_b32 s3, s1, s0
223; GFX8-NEXT:    s_and_b32 s0, s2, s0
224; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
225; GFX8-NEXT:    v_mov_b32_e32 v3, s0
226; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
227; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x80008
228; GFX8-NEXT:    v_mov_b32_e32 v4, s5
229; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
230; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
231; GFX8-NEXT:    v_mov_b32_e32 v5, s7
232; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
233; GFX8-NEXT:    s_waitcnt vmcnt(0)
234; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
235; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v4, v2
236; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
237; GFX8-NEXT:    v_mov_b32_e32 v3, s2
238; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
239; GFX8-NEXT:    flat_store_short v[0:1], v2
240; GFX8-NEXT:    s_endpgm
241;
242; GFX9-NODL-LABEL: udot4_acc16:
243; GFX9-NODL:       ; %bb.0: ; %entry
244; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
245; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
246; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
247; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
248; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
249; GFX9-NODL-NEXT:    global_load_ushort v1, v0, s[0:1]
250; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
251; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
252; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
253; GFX9-NODL-NEXT:    s_and_b32 s5, s3, s2
254; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
255; GFX9-NODL-NEXT:    s_bfe_u32 s7, s4, 0x80008
256; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s2
257; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
258; GFX9-NODL-NEXT:    s_bfe_u32 s6, s3, 0x80008
259; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s7
260; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
261; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
262; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s9
263; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
264; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
265; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s5, v2, v1
266; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s6, v3, v1
267; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s8, v4, v1
268; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
269; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s3, v2, v1
270; GFX9-NODL-NEXT:    global_store_short v0, v1, s[0:1]
271; GFX9-NODL-NEXT:    s_endpgm
272;
273; GFX9-DL-LABEL: udot4_acc16:
274; GFX9-DL:       ; %bb.0: ; %entry
275; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
276; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
277; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
278; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
279; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
280; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
281; GFX9-DL-NEXT:    global_load_ushort v1, v0, s[0:1]
282; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
283; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
284; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
285; GFX9-DL-NEXT:    v_dot4_u32_u8 v1, s2, v2, v1
286; GFX9-DL-NEXT:    global_store_short v0, v1, s[0:1]
287; GFX9-DL-NEXT:    s_endpgm
288;
289; GFX10-DL-LABEL: udot4_acc16:
290; GFX10-DL:       ; %bb.0: ; %entry
291; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
292; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
293; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
294; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
295; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
296; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
297; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
298; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
299; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
300; GFX10-DL-NEXT:    v_dot4_u32_u8 v1, s0, s1, v1
301; GFX10-DL-NEXT:    global_store_short v0, v1, s[4:5]
302; GFX10-DL-NEXT:    s_endpgm
303                                       <4 x i8> addrspace(1)* %src2,
304                                       i16 addrspace(1)* nocapture %dst) {
305entry:
306  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
307  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
308
309  %v1e0 = extractelement <4 x i8> %vec1, i64 0
310  %cv1e0 = zext i8 %v1e0 to i16
311  %v2e0 = extractelement <4 x i8> %vec2, i64 0
312  %cv2e0 = zext i8 %v2e0 to i16
313  %mul1 = mul nuw nsw i16 %cv1e0, %cv2e0
314
315  %v1e1 = extractelement <4 x i8> %vec1, i64 1
316  %cv1e1 = zext i8 %v1e1 to i16
317  %v2e1 = extractelement <4 x i8> %vec2, i64 1
318  %cv2e1 = zext i8 %v2e1 to i16
319  %mul2 = mul nuw nsw i16 %cv1e1, %cv2e1
320
321  %v1e2 = extractelement <4 x i8> %vec1, i64 2
322  %cv1e2 = zext i8 %v1e2 to i16
323  %v2e2 = extractelement <4 x i8> %vec2, i64 2
324  %cv2e2 = zext i8 %v2e2 to i16
325  %mul3 = mul nuw nsw i16 %cv1e2, %cv2e2
326
327  %v1e3 = extractelement <4 x i8> %vec1, i64 3
328  %cv1e3 = zext i8 %v1e3 to i16
329  %v2e3 = extractelement <4 x i8> %vec2, i64 3
330  %cv2e3 = zext i8 %v2e3 to i16
331  %mul4 = mul nuw nsw i16 %cv1e3, %cv2e3
332
333  %acc = load i16, i16 addrspace(1)* %dst, align 2
334  %mad1 = add i16 %mul1, %acc
335  %mad2 = add i16 %mad1, %mul2
336  %mad3 = add i16 %mad2, %mul3
337  %mad4 = add i16 %mad3, %mul4
338
339  store i16 %mad4, i16 addrspace(1)* %dst, align 2
340  ret void
341}
342
343define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1,
344; GFX7-LABEL: udot4_acc8:
345; GFX7:       ; %bb.0: ; %entry
346; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
347; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
348; GFX7-NEXT:    s_mov_b32 s3, 0xf000
349; GFX7-NEXT:    s_mov_b32 s2, -1
350; GFX7-NEXT:    s_movk_i32 s8, 0xff
351; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
352; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
353; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
354; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
355; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
356; GFX7-NEXT:    s_and_b32 s7, s4, s8
357; GFX7-NEXT:    s_and_b32 s6, s5, s8
358; GFX7-NEXT:    s_bfe_u32 s8, s5, 0x80008
359; GFX7-NEXT:    v_mov_b32_e32 v1, s6
360; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80010
361; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
362; GFX7-NEXT:    v_mov_b32_e32 v2, s8
363; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
364; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
365; GFX7-NEXT:    v_mov_b32_e32 v3, s10
366; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
367; GFX7-NEXT:    s_waitcnt vmcnt(0)
368; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
369; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
370; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
371; GFX7-NEXT:    v_mov_b32_e32 v1, s5
372; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
373; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
374; GFX7-NEXT:    s_endpgm
375;
376; GFX8-LABEL: udot4_acc8:
377; GFX8:       ; %bb.0: ; %entry
378; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
379; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
380; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
381; GFX8-NEXT:    v_mov_b32_e32 v0, s0
382; GFX8-NEXT:    v_mov_b32_e32 v1, s1
383; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
384; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
385; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
386; GFX8-NEXT:    s_movk_i32 s0, 0xff
387; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
388; GFX8-NEXT:    s_bfe_u32 s5, s1, 0x80008
389; GFX8-NEXT:    s_and_b32 s3, s2, s0
390; GFX8-NEXT:    s_bfe_u32 s4, s2, 0x80008
391; GFX8-NEXT:    s_and_b32 s0, s1, s0
392; GFX8-NEXT:    v_mov_b32_e32 v3, s3
393; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x80010
394; GFX8-NEXT:    v_mov_b32_e32 v4, s4
395; GFX8-NEXT:    s_bfe_u32 s7, s1, 0x80010
396; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
397; GFX8-NEXT:    v_mov_b32_e32 v5, s6
398; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
399; GFX8-NEXT:    s_waitcnt vmcnt(0)
400; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
401; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
402; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
403; GFX8-NEXT:    v_mov_b32_e32 v3, s2
404; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
405; GFX8-NEXT:    flat_store_byte v[0:1], v2
406; GFX8-NEXT:    s_endpgm
407;
408; GFX9-NODL-LABEL: udot4_acc8:
409; GFX9-NODL:       ; %bb.0: ; %entry
410; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
411; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
412; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
413; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
414; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
415; GFX9-NODL-NEXT:    global_load_ubyte v1, v0, s[0:1]
416; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
417; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
418; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
419; GFX9-NODL-NEXT:    s_bfe_u32 s7, s3, 0x80008
420; GFX9-NODL-NEXT:    s_and_b32 s5, s4, s2
421; GFX9-NODL-NEXT:    s_bfe_u32 s6, s4, 0x80008
422; GFX9-NODL-NEXT:    s_and_b32 s2, s3, s2
423; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
424; GFX9-NODL-NEXT:    s_bfe_u32 s8, s4, 0x80010
425; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s6
426; GFX9-NODL-NEXT:    s_bfe_u32 s9, s3, 0x80010
427; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
428; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s8
429; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
430; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
431; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s2, v2, v1
432; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s7, v3, v1
433; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s9, v4, v1
434; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
435; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s3, v2, v1
436; GFX9-NODL-NEXT:    global_store_byte v0, v1, s[0:1]
437; GFX9-NODL-NEXT:    s_endpgm
438;
439; GFX9-DL-LABEL: udot4_acc8:
440; GFX9-DL:       ; %bb.0: ; %entry
441; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
442; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
443; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
444; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
445; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
446; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
447; GFX9-DL-NEXT:    global_load_ubyte v1, v0, s[0:1]
448; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
449; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
450; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
451; GFX9-DL-NEXT:    v_dot4_u32_u8 v1, s2, v2, v1
452; GFX9-DL-NEXT:    global_store_byte v0, v1, s[0:1]
453; GFX9-DL-NEXT:    s_endpgm
454;
455; GFX10-DL-LABEL: udot4_acc8:
456; GFX10-DL:       ; %bb.0: ; %entry
457; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
458; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
459; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
460; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
461; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
462; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
463; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
464; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
465; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
466; GFX10-DL-NEXT:    v_dot4_u32_u8 v1, s0, s1, v1
467; GFX10-DL-NEXT:    global_store_byte v0, v1, s[4:5]
468; GFX10-DL-NEXT:    s_endpgm
469                                      <4 x i8> addrspace(1)* %src2,
470                                      i8 addrspace(1)* nocapture %dst) {
471entry:
472  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
473  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
474
475  %v1e0 = extractelement <4 x i8> %vec1, i64 0
476  %v2e0 = extractelement <4 x i8> %vec2, i64 0
477  %mul1 = mul nuw nsw i8 %v1e0, %v2e0
478
479  %v1e1 = extractelement <4 x i8> %vec1, i64 1
480  %v2e1 = extractelement <4 x i8> %vec2, i64 1
481  %mul2 = mul nuw nsw i8 %v1e1, %v2e1
482
483  %v1e2 = extractelement <4 x i8> %vec1, i64 2
484  %v2e2 = extractelement <4 x i8> %vec2, i64 2
485  %mul3 = mul nuw nsw i8 %v1e2, %v2e2
486
487  %v1e3 = extractelement <4 x i8> %vec1, i64 3
488  %v2e3 = extractelement <4 x i8> %vec2, i64 3
489  %mul4 = mul nuw nsw i8 %v1e3, %v2e3
490
491  %acc = load i8, i8 addrspace(1)* %dst, align 2
492  %mad1 = add i8 %mul1, %acc
493  %mad2 = add i8 %mad1, %mul2
494  %mad3 = add i8 %mad2, %mul3
495  %mad4 = add i8 %mad3, %mul4
496
497  store i8 %mad4, i8 addrspace(1)* %dst, align 2
498  ret void
499}
500
501; TODO: Generate udot4?
502define amdgpu_kernel void @udot2_8(<4 x i8> addrspace(1)* %src1,
503; GFX7-LABEL: udot2_8:
504; GFX7:       ; %bb.0: ; %entry
505; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
506; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
507; GFX7-NEXT:    s_mov_b32 s3, 0xf000
508; GFX7-NEXT:    s_mov_b32 s2, -1
509; GFX7-NEXT:    s_movk_i32 s8, 0xff
510; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
511; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
512; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
513; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
514; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
515; GFX7-NEXT:    s_and_b32 s7, s4, s8
516; GFX7-NEXT:    s_and_b32 s6, s5, s8
517; GFX7-NEXT:    v_mov_b32_e32 v1, s6
518; GFX7-NEXT:    s_bfe_u32 s5, s5, 0x80008
519; GFX7-NEXT:    s_bfe_u32 s4, s4, 0x80008
520; GFX7-NEXT:    s_waitcnt vmcnt(0)
521; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
522; GFX7-NEXT:    v_mov_b32_e32 v1, s5
523; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
524; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
525; GFX7-NEXT:    s_endpgm
526;
527; GFX8-LABEL: udot2_8:
528; GFX8:       ; %bb.0: ; %entry
529; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
530; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
531; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
532; GFX8-NEXT:    v_mov_b32_e32 v0, s0
533; GFX8-NEXT:    v_mov_b32_e32 v1, s1
534; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
535; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
536; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
537; GFX8-NEXT:    s_movk_i32 s0, 0xff
538; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
539; GFX8-NEXT:    s_and_b32 s3, s2, s0
540; GFX8-NEXT:    s_and_b32 s0, s1, s0
541; GFX8-NEXT:    v_mov_b32_e32 v3, s3
542; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x80008
543; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x80008
544; GFX8-NEXT:    s_waitcnt vmcnt(0)
545; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
546; GFX8-NEXT:    v_mov_b32_e32 v3, s2
547; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
548; GFX8-NEXT:    flat_store_byte v[0:1], v2
549; GFX8-NEXT:    s_endpgm
550;
551; GFX9-NODL-LABEL: udot2_8:
552; GFX9-NODL:       ; %bb.0: ; %entry
553; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
554; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
555; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
556; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
557; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
558; GFX9-NODL-NEXT:    global_load_ubyte v1, v0, s[0:1]
559; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
560; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
561; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
562; GFX9-NODL-NEXT:    s_and_b32 s5, s4, s2
563; GFX9-NODL-NEXT:    s_and_b32 s2, s3, s2
564; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
565; GFX9-NODL-NEXT:    s_bfe_u32 s4, s4, 0x80008
566; GFX9-NODL-NEXT:    s_bfe_u32 s3, s3, 0x80008
567; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
568; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s2, v2, v1
569; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
570; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s3, v2, v1
571; GFX9-NODL-NEXT:    global_store_byte v0, v1, s[0:1]
572; GFX9-NODL-NEXT:    s_endpgm
573;
574; GFX9-DL-LABEL: udot2_8:
575; GFX9-DL:       ; %bb.0: ; %entry
576; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
577; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
578; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
579; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
580; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
581; GFX9-DL-NEXT:    global_load_ubyte v1, v0, s[0:1]
582; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
583; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
584; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
585; GFX9-DL-NEXT:    s_and_b32 s5, s4, s2
586; GFX9-DL-NEXT:    s_and_b32 s2, s3, s2
587; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
588; GFX9-DL-NEXT:    s_bfe_u32 s4, s4, 0x80008
589; GFX9-DL-NEXT:    s_bfe_u32 s3, s3, 0x80008
590; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
591; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s2, v2, v1
592; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
593; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s3, v2, v1
594; GFX9-DL-NEXT:    global_store_byte v0, v1, s[0:1]
595; GFX9-DL-NEXT:    s_endpgm
596;
597; GFX10-DL-LABEL: udot2_8:
598; GFX10-DL:       ; %bb.0: ; %entry
599; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
600; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
601; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
602; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
603; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
604; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
605; GFX10-DL-NEXT:    s_load_dword s2, s[2:3], 0x0
606; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
607; GFX10-DL-NEXT:    s_movk_i32 s1, 0xff
608; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
609; GFX10-DL-NEXT:    s_and_b32 s3, s2, s1
610; GFX10-DL-NEXT:    s_and_b32 s1, s0, s1
611; GFX10-DL-NEXT:    s_bfe_u32 s0, s0, 0x80008
612; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
613; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s1, s3, v1
614; GFX10-DL-NEXT:    s_bfe_u32 s1, s2, 0x80008
615; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s0, s1, v1
616; GFX10-DL-NEXT:    global_store_byte v0, v1, s[4:5]
617; GFX10-DL-NEXT:    s_endpgm
618                                   <4 x i8> addrspace(1)* %src2,
619                                   i8 addrspace(1)* nocapture %dst) {
620entry:
621  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
622  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
623
624  %v1e0 = extractelement <4 x i8> %vec1, i64 0
625  %v2e0 = extractelement <4 x i8> %vec2, i64 0
626  %mul1 = mul nuw nsw i8 %v1e0, %v2e0
627
628  %v1e1 = extractelement <4 x i8> %vec1, i64 1
629  %v2e1 = extractelement <4 x i8> %vec2, i64 1
630  %mul2 = mul nuw nsw i8 %v1e1, %v2e1
631
632  %acc = load i8, i8 addrspace(1)* %dst, align 2
633  %mad1 = add i8 %mul1, %acc
634  %mad2 = add i8 %mad1, %mul2
635  store i8 %mad2, i8 addrspace(1)* %dst, align 2
636  ret void
637}
638
639define amdgpu_kernel void @udot4_CommutationInsideMAD(<4 x i8> addrspace(1)* %src1,
640; GFX7-LABEL: udot4_CommutationInsideMAD:
641; GFX7:       ; %bb.0: ; %entry
642; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
643; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
644; GFX7-NEXT:    s_mov_b32 s3, 0xf000
645; GFX7-NEXT:    s_mov_b32 s2, -1
646; GFX7-NEXT:    s_movk_i32 s8, 0xff
647; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
648; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
649; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
650; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
651; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
652; GFX7-NEXT:    s_and_b32 s6, s4, s8
653; GFX7-NEXT:    s_and_b32 s7, s5, s8
654; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
655; GFX7-NEXT:    v_mov_b32_e32 v1, s6
656; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
657; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
658; GFX7-NEXT:    v_mov_b32_e32 v2, s8
659; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
660; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
661; GFX7-NEXT:    v_mov_b32_e32 v3, s10
662; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
663; GFX7-NEXT:    s_waitcnt vmcnt(0)
664; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
665; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
666; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
667; GFX7-NEXT:    v_mov_b32_e32 v1, s4
668; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
669; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
670; GFX7-NEXT:    s_endpgm
671;
672; GFX8-LABEL: udot4_CommutationInsideMAD:
673; GFX8:       ; %bb.0: ; %entry
674; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
675; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
676; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
677; GFX8-NEXT:    v_mov_b32_e32 v0, s0
678; GFX8-NEXT:    v_mov_b32_e32 v1, s1
679; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
680; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
681; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
682; GFX8-NEXT:    s_movk_i32 s0, 0xff
683; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
684; GFX8-NEXT:    s_and_b32 s3, s1, s0
685; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x80008
686; GFX8-NEXT:    s_and_b32 s0, s2, s0
687; GFX8-NEXT:    v_mov_b32_e32 v3, s3
688; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
689; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
690; GFX8-NEXT:    v_mov_b32_e32 v4, s4
691; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
692; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
693; GFX8-NEXT:    v_mov_b32_e32 v5, s6
694; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
695; GFX8-NEXT:    s_waitcnt vmcnt(0)
696; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
697; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v4, v2
698; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
699; GFX8-NEXT:    v_mov_b32_e32 v3, s1
700; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
701; GFX8-NEXT:    flat_store_byte v[0:1], v2
702; GFX8-NEXT:    s_endpgm
703;
704; GFX9-NODL-LABEL: udot4_CommutationInsideMAD:
705; GFX9-NODL:       ; %bb.0: ; %entry
706; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
707; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
708; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
709; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
710; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
711; GFX9-NODL-NEXT:    global_load_ubyte v1, v0, s[0:1]
712; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
713; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
714; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
715; GFX9-NODL-NEXT:    s_and_b32 s5, s3, s2
716; GFX9-NODL-NEXT:    s_bfe_u32 s6, s3, 0x80008
717; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
718; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
719; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
720; GFX9-NODL-NEXT:    s_bfe_u32 s7, s4, 0x80008
721; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s6
722; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
723; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
724; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s8
725; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
726; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
727; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s2, v2, v1
728; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s7, v3, v1
729; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s9, v4, v1
730; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s3
731; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s4, v2, v1
732; GFX9-NODL-NEXT:    global_store_byte v0, v1, s[0:1]
733; GFX9-NODL-NEXT:    s_endpgm
734;
735; GFX9-DL-LABEL: udot4_CommutationInsideMAD:
736; GFX9-DL:       ; %bb.0: ; %entry
737; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
738; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
739; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
740; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
741; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
742; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
743; GFX9-DL-NEXT:    global_load_ubyte v1, v0, s[0:1]
744; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
745; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
746; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
747; GFX9-DL-NEXT:    v_dot4_u32_u8 v1, s3, v2, v1
748; GFX9-DL-NEXT:    global_store_byte v0, v1, s[0:1]
749; GFX9-DL-NEXT:    s_endpgm
750;
751; GFX10-DL-LABEL: udot4_CommutationInsideMAD:
752; GFX10-DL:       ; %bb.0: ; %entry
753; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
754; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
755; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
756; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
757; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
758; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
759; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
760; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
761; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
762; GFX10-DL-NEXT:    v_dot4_u32_u8 v1, s1, s0, v1
763; GFX10-DL-NEXT:    global_store_byte v0, v1, s[4:5]
764; GFX10-DL-NEXT:    s_endpgm
765                                                      <4 x i8> addrspace(1)* %src2,
766                                                      i8 addrspace(1)* nocapture %dst) {
767entry:
768  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
769  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
770
771  %v1e0 = extractelement <4 x i8> %vec1, i64 0
772  %v2e0 = extractelement <4 x i8> %vec2, i64 0
773  %mul1 = mul nuw nsw i8 %v2e0, %v1e0
774
775  %v1e1 = extractelement <4 x i8> %vec1, i64 1
776  %v2e1 = extractelement <4 x i8> %vec2, i64 1
777  %mul2 = mul nuw nsw i8 %v2e1, %v1e1
778
779  %v1e2 = extractelement <4 x i8> %vec1, i64 2
780  %v2e2 = extractelement <4 x i8> %vec2, i64 2
781  %mul3 = mul nuw nsw i8 %v2e2, %v1e2
782
783  %v1e3 = extractelement <4 x i8> %vec1, i64 3
784  %v2e3 = extractelement <4 x i8> %vec2, i64 3
785  %mul4 = mul nuw nsw i8 %v2e3, %v1e3
786
787  %acc = load i8, i8 addrspace(1)* %dst, align 2
788  %mad1 = add i8 %acc, %mul1
789  %mad2 = add i8 %mul2, %mad1
790  %mad3 = add i8 %mul3, %mad2
791  %mad4 = add i8 %mul4, %mad3
792
793  store i8 %mad4, i8 addrspace(1)* %dst, align 2
794  ret void
795}
796
797; TODO: Support commutation accross the adds.
798define amdgpu_kernel void @udot4_CommutationAccrossMADs(<4 x i8> addrspace(1)* %src1,
799; GFX7-LABEL: udot4_CommutationAccrossMADs:
800; GFX7:       ; %bb.0: ; %entry
801; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
802; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
803; GFX7-NEXT:    s_mov_b32 s3, 0xf000
804; GFX7-NEXT:    s_mov_b32 s2, -1
805; GFX7-NEXT:    s_movk_i32 s8, 0xff
806; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
807; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
808; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
809; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
810; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
811; GFX7-NEXT:    s_and_b32 s6, s4, s8
812; GFX7-NEXT:    s_and_b32 s7, s5, s8
813; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
814; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
815; GFX7-NEXT:    v_mov_b32_e32 v1, s8
816; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
817; GFX7-NEXT:    v_mov_b32_e32 v2, s6
818; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
819; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
820; GFX7-NEXT:    v_mov_b32_e32 v3, s10
821; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
822; GFX7-NEXT:    s_waitcnt vmcnt(0)
823; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
824; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v2, v0
825; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
826; GFX7-NEXT:    v_mov_b32_e32 v1, s4
827; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
828; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
829; GFX7-NEXT:    s_endpgm
830;
831; GFX8-LABEL: udot4_CommutationAccrossMADs:
832; GFX8:       ; %bb.0: ; %entry
833; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
834; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
835; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
836; GFX8-NEXT:    v_mov_b32_e32 v0, s0
837; GFX8-NEXT:    v_mov_b32_e32 v1, s1
838; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
839; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
840; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
841; GFX8-NEXT:    s_movk_i32 s0, 0xff
842; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
843; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x80008
844; GFX8-NEXT:    s_and_b32 s3, s1, s0
845; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x80008
846; GFX8-NEXT:    v_mov_b32_e32 v3, s4
847; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
848; GFX8-NEXT:    s_and_b32 s0, s2, s0
849; GFX8-NEXT:    v_mov_b32_e32 v4, s3
850; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
851; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
852; GFX8-NEXT:    v_mov_b32_e32 v5, s6
853; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
854; GFX8-NEXT:    s_waitcnt vmcnt(0)
855; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v3, v2
856; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v4, v2
857; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
858; GFX8-NEXT:    v_mov_b32_e32 v3, s1
859; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
860; GFX8-NEXT:    flat_store_byte v[0:1], v2
861; GFX8-NEXT:    s_endpgm
862;
863; GFX9-NODL-LABEL: udot4_CommutationAccrossMADs:
864; GFX9-NODL:       ; %bb.0: ; %entry
865; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
866; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
867; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
868; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
869; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
870; GFX9-NODL-NEXT:    global_load_ubyte v1, v0, s[0:1]
871; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
872; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
873; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
874; GFX9-NODL-NEXT:    s_bfe_u32 s6, s3, 0x80008
875; GFX9-NODL-NEXT:    s_and_b32 s5, s3, s2
876; GFX9-NODL-NEXT:    s_bfe_u32 s7, s4, 0x80008
877; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s6
878; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
879; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
880; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s5
881; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
882; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
883; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s8
884; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
885; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
886; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s7, v2, v1
887; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s2, v3, v1
888; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s9, v4, v1
889; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s3
890; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s4, v2, v1
891; GFX9-NODL-NEXT:    global_store_byte v0, v1, s[0:1]
892; GFX9-NODL-NEXT:    s_endpgm
893;
894; GFX9-DL-LABEL: udot4_CommutationAccrossMADs:
895; GFX9-DL:       ; %bb.0: ; %entry
896; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
897; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
898; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
899; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
900; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
901; GFX9-DL-NEXT:    global_load_ubyte v1, v0, s[0:1]
902; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
903; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
904; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
905; GFX9-DL-NEXT:    s_bfe_u32 s6, s3, 0x80008
906; GFX9-DL-NEXT:    s_and_b32 s5, s3, s2
907; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x80008
908; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s6
909; GFX9-DL-NEXT:    s_bfe_u32 s8, s3, 0x80010
910; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
911; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
912; GFX9-DL-NEXT:    s_bfe_u32 s9, s4, 0x80010
913; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 24
914; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s8
915; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 24
916; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
917; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s7, v2, v1
918; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s2, v3, v1
919; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s9, v4, v1
920; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
921; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s4, v2, v1
922; GFX9-DL-NEXT:    global_store_byte v0, v1, s[0:1]
923; GFX9-DL-NEXT:    s_endpgm
924;
925; GFX10-DL-LABEL: udot4_CommutationAccrossMADs:
926; GFX10-DL:       ; %bb.0: ; %entry
927; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
928; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
929; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
930; GFX10-DL-NEXT:    s_movk_i32 s6, 0xff
931; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
932; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
933; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
934; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
935; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
936; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
937; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x80008
938; GFX10-DL-NEXT:    s_bfe_u32 s3, s1, 0x80008
939; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
940; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s3, s2, v1
941; GFX10-DL-NEXT:    s_and_b32 s2, s0, s6
942; GFX10-DL-NEXT:    s_and_b32 s3, s1, s6
943; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s3, s2, v1
944; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x80010
945; GFX10-DL-NEXT:    s_bfe_u32 s3, s1, 0x80010
946; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 24
947; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 24
948; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s3, s2, v1
949; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s1, s0, v1
950; GFX10-DL-NEXT:    global_store_byte v0, v1, s[4:5]
951; GFX10-DL-NEXT:    s_endpgm
952                                                        <4 x i8> addrspace(1)* %src2,
953                                                        i8 addrspace(1)* nocapture %dst) {
954entry:
955  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
956  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
957
958  %v1e0 = extractelement <4 x i8> %vec1, i64 0
959  %v2e0 = extractelement <4 x i8> %vec2, i64 0
960  %mul1 = mul nuw nsw i8 %v2e0, %v1e0
961
962  %v1e1 = extractelement <4 x i8> %vec1, i64 1
963  %v2e1 = extractelement <4 x i8> %vec2, i64 1
964  %mul2 = mul nuw nsw i8 %v2e1, %v1e1
965
966  %v1e2 = extractelement <4 x i8> %vec1, i64 2
967  %v2e2 = extractelement <4 x i8> %vec2, i64 2
968  %mul3 = mul nuw nsw i8 %v2e2, %v1e2
969
970  %v1e3 = extractelement <4 x i8> %vec1, i64 3
971  %v2e3 = extractelement <4 x i8> %vec2, i64 3
972  %mul4 = mul nuw nsw i8 %v2e3, %v1e3
973
974  %acc = load i8, i8 addrspace(1)* %dst, align 2
975  %mad1 = add i8 %acc, %mul2
976  %mad2 = add i8 %mad1, %mul1
977  %mad3 = add i8 %mad2, %mul3
978  %mad4 = add i8 %mad3, %mul4
979
980  store i8 %mad4, i8 addrspace(1)* %dst, align 2
981  ret void
982}
983
984define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
985; GFX7-LABEL: udot4_multiuse_mul1:
986; GFX7:       ; %bb.0: ; %entry
987; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
988; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
989; GFX7-NEXT:    s_movk_i32 s8, 0xff
990; GFX7-NEXT:    s_mov_b32 s3, 0xf000
991; GFX7-NEXT:    s_mov_b32 s2, -1
992; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
993; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
994; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
995; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
996; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
997; GFX7-NEXT:    s_and_b32 s6, s4, s8
998; GFX7-NEXT:    s_and_b32 s7, s5, s8
999; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
1000; GFX7-NEXT:    v_mov_b32_e32 v0, s7
1001; GFX7-NEXT:    v_mov_b32_e32 v1, s12
1002; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
1003; GFX7-NEXT:    v_mad_u32_u24 v1, s6, v0, v1
1004; GFX7-NEXT:    v_mov_b32_e32 v2, s9
1005; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
1006; GFX7-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
1007; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
1008; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
1009; GFX7-NEXT:    v_mov_b32_e32 v1, s11
1010; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
1011; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
1012; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
1013; GFX7-NEXT:    v_mov_b32_e32 v1, s5
1014; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1015; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1016; GFX7-NEXT:    s_endpgm
1017;
1018; GFX8-LABEL: udot4_multiuse_mul1:
1019; GFX8:       ; %bb.0: ; %entry
1020; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1021; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1022; GFX8-NEXT:    s_movk_i32 s2, 0xff
1023; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1024; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1025; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1026; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
1027; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1028; GFX8-NEXT:    s_and_b32 s5, s3, s2
1029; GFX8-NEXT:    s_and_b32 s2, s4, s2
1030; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x80008
1031; GFX8-NEXT:    v_mov_b32_e32 v0, s2
1032; GFX8-NEXT:    v_mov_b32_e32 v1, s10
1033; GFX8-NEXT:    s_bfe_u32 s6, s3, 0x80008
1034; GFX8-NEXT:    v_mad_u32_u24 v1, s5, v0, v1
1035; GFX8-NEXT:    v_mov_b32_e32 v2, s7
1036; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x80010
1037; GFX8-NEXT:    v_mad_u32_u24 v1, s6, v2, v1
1038; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x80010
1039; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1040; GFX8-NEXT:    v_mov_b32_e32 v1, s9
1041; GFX8-NEXT:    s_lshr_b32 s4, s4, 24
1042; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
1043; GFX8-NEXT:    s_lshr_b32 s3, s3, 24
1044; GFX8-NEXT:    v_mov_b32_e32 v1, s4
1045; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v1, v0
1046; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1047; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1048; GFX8-NEXT:    flat_store_dword v[0:1], v2
1049; GFX8-NEXT:    s_endpgm
1050;
1051; GFX9-NODL-LABEL: udot4_multiuse_mul1:
1052; GFX9-NODL:       ; %bb.0: ; %entry
1053; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1054; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1055; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
1056; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
1057; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1058; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1059; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1060; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
1061; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1062; GFX9-NODL-NEXT:    s_and_b32 s5, s3, s2
1063; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1064; GFX9-NODL-NEXT:    s_bfe_u32 s7, s4, 0x80008
1065; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
1066; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s10
1067; GFX9-NODL-NEXT:    s_bfe_u32 s6, s3, 0x80008
1068; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s5, v1, v2
1069; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s7
1070; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
1071; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s6, v3, v2
1072; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
1073; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s5, v1, v2
1074; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s9
1075; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
1076; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
1077; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
1078; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
1079; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s3, v2, v1
1080; GFX9-NODL-NEXT:    global_store_dword v0, v1, s[0:1]
1081; GFX9-NODL-NEXT:    s_endpgm
1082;
1083; GFX9-DL-LABEL: udot4_multiuse_mul1:
1084; GFX9-DL:       ; %bb.0: ; %entry
1085; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1086; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1087; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
1088; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
1089; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1090; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1091; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1092; GFX9-DL-NEXT:    s_load_dword s10, s[0:1], 0x0
1093; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1094; GFX9-DL-NEXT:    s_and_b32 s5, s3, s2
1095; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
1096; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x80008
1097; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
1098; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s10
1099; GFX9-DL-NEXT:    s_bfe_u32 s6, s3, 0x80008
1100; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s5, v1, v2
1101; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s7
1102; GFX9-DL-NEXT:    s_bfe_u32 s9, s4, 0x80010
1103; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s6, v3, v2
1104; GFX9-DL-NEXT:    s_bfe_u32 s8, s3, 0x80010
1105; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s5, v1, v2
1106; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s9
1107; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 24
1108; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s8, v2, v1
1109; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 24
1110; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
1111; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s3, v2, v1
1112; GFX9-DL-NEXT:    global_store_dword v0, v1, s[0:1]
1113; GFX9-DL-NEXT:    s_endpgm
1114;
1115; GFX10-DL-LABEL: udot4_multiuse_mul1:
1116; GFX10-DL:       ; %bb.0: ; %entry
1117; GFX10-DL-NEXT:    s_clause 0x1
1118; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1119; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1120; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
1121; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1122; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1123; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
1124; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
1125; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
1126; GFX10-DL-NEXT:    s_movk_i32 s2, 0xff
1127; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1128; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1129; GFX10-DL-NEXT:    s_and_b32 s3, s0, s2
1130; GFX10-DL-NEXT:    s_and_b32 s2, s1, s2
1131; GFX10-DL-NEXT:    s_bfe_u32 s6, s0, 0x80008
1132; GFX10-DL-NEXT:    s_bfe_u32 s7, s1, 0x80008
1133; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s3, s2, v0
1134; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s7, v0
1135; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s3, s2, v0
1136; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x80010
1137; GFX10-DL-NEXT:    s_bfe_u32 s3, s1, 0x80010
1138; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 24
1139; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 24
1140; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s2, s3, v0
1141; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s0, s1, v0
1142; GFX10-DL-NEXT:    global_store_dword v1, v0, s[4:5]
1143; GFX10-DL-NEXT:    s_endpgm
1144                                               <4 x i8> addrspace(1)* %src2,
1145                                               i32 addrspace(1)* nocapture %dst) {
1146entry:
1147  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
1148  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
1149
1150  %v1e0 = extractelement <4 x i8> %vec1, i64 0
1151  %cv1e0 = zext i8 %v1e0 to i32
1152  %v2e0 = extractelement <4 x i8> %vec2, i64 0
1153  %cv2e0 = zext i8 %v2e0 to i32
1154  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
1155
1156  %v1e1 = extractelement <4 x i8> %vec1, i64 1
1157  %cv1e1 = zext i8 %v1e1 to i32
1158  %v2e1 = extractelement <4 x i8> %vec2, i64 1
1159  %cv2e1 = zext i8 %v2e1 to i32
1160  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
1161
1162  %v1e2 = extractelement <4 x i8> %vec1, i64 2
1163  %cv1e2 = zext i8 %v1e2 to i32
1164  %v2e2 = extractelement <4 x i8> %vec2, i64 2
1165  %cv2e2 = zext i8 %v2e2 to i32
1166  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
1167
1168  %v1e3 = extractelement <4 x i8> %vec1, i64 3
1169  %cv1e3 = zext i8 %v1e3 to i32
1170  %v2e3 = extractelement <4 x i8> %vec2, i64 3
1171  %cv2e3 = zext i8 %v2e3 to i32
1172  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
1173
1174  %acc = load i32, i32 addrspace(1)* %dst, align 4
1175  %add = add i32 %mul1, %acc
1176  %add1 = add i32 %mul2, %add
1177  %add2 = add i32 %add1, %mul1
1178  %add3 = add i32 %add2, %mul3
1179  %add4 = add i32 %add3, %mul4
1180
1181  store i32 %add4, i32 addrspace(1)* %dst, align 4
1182  ret void
1183}
1184
1185define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
1186; GFX7-LABEL: udot4_multiuse_add1:
1187; GFX7:       ; %bb.0: ; %entry
1188; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1189; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1190; GFX7-NEXT:    s_movk_i32 s8, 0xff
1191; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1192; GFX7-NEXT:    s_mov_b32 s2, -1
1193; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1194; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1195; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1196; GFX7-NEXT:    s_load_dword s12, s[0:1], 0x0
1197; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1198; GFX7-NEXT:    s_and_b32 s6, s4, s8
1199; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
1200; GFX7-NEXT:    s_and_b32 s7, s5, s8
1201; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x80008
1202; GFX7-NEXT:    v_mov_b32_e32 v0, s9
1203; GFX7-NEXT:    v_mov_b32_e32 v1, s12
1204; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v0, v1
1205; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
1206; GFX7-NEXT:    v_mov_b32_e32 v2, s7
1207; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
1208; GFX7-NEXT:    v_add_i32_e32 v1, vcc, s12, v0
1209; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v2, v0
1210; GFX7-NEXT:    v_mov_b32_e32 v2, s11
1211; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
1212; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v2, v0
1213; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
1214; GFX7-NEXT:    v_mov_b32_e32 v2, s5
1215; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v2, v0
1216; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
1217; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1218; GFX7-NEXT:    s_endpgm
1219;
1220; GFX8-LABEL: udot4_multiuse_add1:
1221; GFX8:       ; %bb.0: ; %entry
1222; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1223; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1224; GFX8-NEXT:    s_movk_i32 s2, 0xff
1225; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1226; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1227; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1228; GFX8-NEXT:    s_load_dword s10, s[0:1], 0x0
1229; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1230; GFX8-NEXT:    s_and_b32 s5, s3, s2
1231; GFX8-NEXT:    s_bfe_u32 s7, s4, 0x80008
1232; GFX8-NEXT:    s_and_b32 s2, s4, s2
1233; GFX8-NEXT:    s_bfe_u32 s6, s3, 0x80008
1234; GFX8-NEXT:    v_mov_b32_e32 v0, s7
1235; GFX8-NEXT:    v_mov_b32_e32 v1, s10
1236; GFX8-NEXT:    v_mad_u32_u24 v0, s6, v0, v1
1237; GFX8-NEXT:    s_bfe_u32 s9, s4, 0x80010
1238; GFX8-NEXT:    v_mov_b32_e32 v2, s2
1239; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x80010
1240; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s10, v0
1241; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v2, v0
1242; GFX8-NEXT:    v_mov_b32_e32 v2, s9
1243; GFX8-NEXT:    s_lshr_b32 s4, s4, 24
1244; GFX8-NEXT:    v_mad_u32_u24 v0, s8, v2, v0
1245; GFX8-NEXT:    s_lshr_b32 s3, s3, 24
1246; GFX8-NEXT:    v_mov_b32_e32 v2, s4
1247; GFX8-NEXT:    v_mad_u32_u24 v0, s3, v2, v0
1248; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v1
1249; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1250; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1251; GFX8-NEXT:    flat_store_dword v[0:1], v2
1252; GFX8-NEXT:    s_endpgm
1253;
1254; GFX9-NODL-LABEL: udot4_multiuse_add1:
1255; GFX9-NODL:       ; %bb.0: ; %entry
1256; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1257; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1258; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
1259; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
1260; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1261; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1262; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1263; GFX9-NODL-NEXT:    s_load_dword s10, s[0:1], 0x0
1264; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1265; GFX9-NODL-NEXT:    s_and_b32 s5, s3, s2
1266; GFX9-NODL-NEXT:    s_bfe_u32 s7, s4, 0x80008
1267; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1268; GFX9-NODL-NEXT:    s_bfe_u32 s6, s3, 0x80008
1269; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s7
1270; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s10
1271; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s6, v1, v2
1272; GFX9-NODL-NEXT:    s_bfe_u32 s9, s4, 0x80010
1273; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
1274; GFX9-NODL-NEXT:    s_bfe_u32 s8, s3, 0x80010
1275; GFX9-NODL-NEXT:    v_add_u32_e32 v2, s10, v1
1276; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s5, v3, v1
1277; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s9
1278; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 24
1279; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s8, v3, v1
1280; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
1281; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s4
1282; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s3, v3, v1
1283; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v1, v2
1284; GFX9-NODL-NEXT:    global_store_dword v0, v1, s[0:1]
1285; GFX9-NODL-NEXT:    s_endpgm
1286;
1287; GFX9-DL-LABEL: udot4_multiuse_add1:
1288; GFX9-DL:       ; %bb.0: ; %entry
1289; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1290; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1291; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
1292; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
1293; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1294; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1295; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1296; GFX9-DL-NEXT:    s_load_dword s10, s[0:1], 0x0
1297; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1298; GFX9-DL-NEXT:    s_and_b32 s5, s3, s2
1299; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x80008
1300; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
1301; GFX9-DL-NEXT:    s_bfe_u32 s6, s3, 0x80008
1302; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s7
1303; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s10
1304; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s6, v1, v2
1305; GFX9-DL-NEXT:    s_bfe_u32 s9, s4, 0x80010
1306; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
1307; GFX9-DL-NEXT:    s_bfe_u32 s8, s3, 0x80010
1308; GFX9-DL-NEXT:    v_add_u32_e32 v2, s10, v1
1309; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s5, v3, v1
1310; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s9
1311; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 24
1312; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s8, v3, v1
1313; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 24
1314; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
1315; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s3, v3, v1
1316; GFX9-DL-NEXT:    v_add_u32_e32 v1, v1, v2
1317; GFX9-DL-NEXT:    global_store_dword v0, v1, s[0:1]
1318; GFX9-DL-NEXT:    s_endpgm
1319;
1320; GFX10-DL-LABEL: udot4_multiuse_add1:
1321; GFX10-DL:       ; %bb.0: ; %entry
1322; GFX10-DL-NEXT:    s_clause 0x1
1323; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1324; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1325; GFX10-DL-NEXT:    s_movk_i32 s7, 0xff
1326; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0
1327; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1328; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1329; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
1330; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
1331; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
1332; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1333; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1334; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x80008
1335; GFX10-DL-NEXT:    s_bfe_u32 s3, s1, 0x80008
1336; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s2, s3, v0
1337; GFX10-DL-NEXT:    s_and_b32 s2, s0, s7
1338; GFX10-DL-NEXT:    s_and_b32 s3, s1, s7
1339; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s2, s3, v0
1340; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x80010
1341; GFX10-DL-NEXT:    s_bfe_u32 s3, s1, 0x80010
1342; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 24
1343; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 24
1344; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s2, s3, v1
1345; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, s6, v0
1346; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s0, s1, v1
1347; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v1, v0
1348; GFX10-DL-NEXT:    global_store_dword v2, v0, s[4:5]
1349; GFX10-DL-NEXT:    s_endpgm
1350                                               <4 x i8> addrspace(1)* %src2,
1351                                               i32 addrspace(1)* nocapture %dst) {
1352entry:
1353  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
1354  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
1355
1356  %v1e0 = extractelement <4 x i8> %vec1, i64 0
1357  %cv1e0 = zext i8 %v1e0 to i32
1358  %v2e0 = extractelement <4 x i8> %vec2, i64 0
1359  %cv2e0 = zext i8 %v2e0 to i32
1360  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
1361
1362  %v1e1 = extractelement <4 x i8> %vec1, i64 1
1363  %cv1e1 = zext i8 %v1e1 to i32
1364  %v2e1 = extractelement <4 x i8> %vec2, i64 1
1365  %cv2e1 = zext i8 %v2e1 to i32
1366  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
1367
1368  %v1e2 = extractelement <4 x i8> %vec1, i64 2
1369  %cv1e2 = zext i8 %v1e2 to i32
1370  %v2e2 = extractelement <4 x i8> %vec2, i64 2
1371  %cv2e2 = zext i8 %v2e2 to i32
1372  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
1373
1374  %v1e3 = extractelement <4 x i8> %vec1, i64 3
1375  %cv1e3 = zext i8 %v1e3 to i32
1376  %v2e3 = extractelement <4 x i8> %vec2, i64 3
1377  %cv2e3 = zext i8 %v2e3 to i32
1378  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
1379
1380  %acc = load i32, i32 addrspace(1)* %dst, align 4
1381  %add1 = add i32 %mul2, %acc
1382  %add = add i32 %add1, %acc
1383  %add2 = add i32 %add1, %mul1
1384  %add3 = add i32 %add2, %mul3
1385  %add4 = add i32 %add3, %mul4
1386  %res = add i32 %add4, %add
1387  store i32 %res, i32 addrspace(1)* %dst, align 4
1388  ret void
1389}
1390
1391define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1,
1392; GFX7-LABEL: notdot4_mixedtypes:
1393; GFX7:       ; %bb.0: ; %entry
1394; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1395; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1396; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1397; GFX7-NEXT:    s_mov_b32 s2, -1
1398; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1399; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1400; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
1401; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1402; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1403; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1404; GFX7-NEXT:    s_sext_i32_i8 s6, s4
1405; GFX7-NEXT:    s_sext_i32_i8 s7, s5
1406; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
1407; GFX7-NEXT:    s_and_b32 s7, s7, s8
1408; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80008
1409; GFX7-NEXT:    v_mov_b32_e32 v1, s9
1410; GFX7-NEXT:    s_bfe_u32 s11, s5, 0x80010
1411; GFX7-NEXT:    s_and_b32 s6, s6, s8
1412; GFX7-NEXT:    v_mov_b32_e32 v3, s7
1413; GFX7-NEXT:    s_bfe_u32 s12, s4, 0x80010
1414; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
1415; GFX7-NEXT:    v_mov_b32_e32 v2, s11
1416; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
1417; GFX7-NEXT:    s_waitcnt vmcnt(0)
1418; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
1419; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v3, v0
1420; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v2, v0
1421; GFX7-NEXT:    v_mov_b32_e32 v1, s5
1422; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1423; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
1424; GFX7-NEXT:    s_endpgm
1425;
1426; GFX8-LABEL: notdot4_mixedtypes:
1427; GFX8:       ; %bb.0: ; %entry
1428; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1429; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1430; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1431; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1432; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1433; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
1434; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
1435; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
1436; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1437; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x80008
1438; GFX8-NEXT:    s_bfe_u32 s5, s1, 0x80008
1439; GFX8-NEXT:    s_sext_i32_i8 s3, s1
1440; GFX8-NEXT:    v_mov_b32_e32 v3, s5
1441; GFX8-NEXT:    s_bfe_u32 s7, s1, 0x80010
1442; GFX8-NEXT:    s_sext_i32_i8 s2, s0
1443; GFX8-NEXT:    v_mov_b32_e32 v4, s3
1444; GFX8-NEXT:    s_bfe_u32 s6, s0, 0x80010
1445; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
1446; GFX8-NEXT:    v_mov_b32_e32 v5, s7
1447; GFX8-NEXT:    s_lshr_b32 s0, s0, 24
1448; GFX8-NEXT:    s_waitcnt vmcnt(0)
1449; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v3, v2
1450; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v4, v2
1451; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
1452; GFX8-NEXT:    v_mov_b32_e32 v3, s1
1453; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
1454; GFX8-NEXT:    flat_store_short v[0:1], v2
1455; GFX8-NEXT:    s_endpgm
1456;
1457; GFX9-NODL-LABEL: notdot4_mixedtypes:
1458; GFX9-NODL:       ; %bb.0: ; %entry
1459; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1460; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1461; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
1462; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1463; GFX9-NODL-NEXT:    global_load_ushort v1, v0, s[0:1]
1464; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
1465; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
1466; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1467; GFX9-NODL-NEXT:    s_bfe_u32 s6, s2, 0x80008
1468; GFX9-NODL-NEXT:    s_bfe_u32 s7, s3, 0x80008
1469; GFX9-NODL-NEXT:    s_sext_i32_i8 s5, s3
1470; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s7
1471; GFX9-NODL-NEXT:    s_bfe_u32 s9, s3, 0x80010
1472; GFX9-NODL-NEXT:    s_sext_i32_i8 s4, s2
1473; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s5
1474; GFX9-NODL-NEXT:    s_bfe_u32 s8, s2, 0x80010
1475; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 24
1476; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s9
1477; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
1478; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
1479; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s6, v2, v1
1480; GFX9-NODL-NEXT:    v_mad_i32_i24 v1, s4, v3, v1
1481; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s8, v4, v1
1482; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s3
1483; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s2, v2, v1
1484; GFX9-NODL-NEXT:    global_store_short v0, v1, s[0:1]
1485; GFX9-NODL-NEXT:    s_endpgm
1486;
1487; GFX9-DL-LABEL: notdot4_mixedtypes:
1488; GFX9-DL:       ; %bb.0: ; %entry
1489; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1490; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1491; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
1492; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1493; GFX9-DL-NEXT:    global_load_ushort v1, v0, s[0:1]
1494; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1495; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1496; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1497; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x80008
1498; GFX9-DL-NEXT:    s_bfe_u32 s7, s3, 0x80008
1499; GFX9-DL-NEXT:    s_sext_i32_i8 s5, s3
1500; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s7
1501; GFX9-DL-NEXT:    s_bfe_u32 s9, s3, 0x80010
1502; GFX9-DL-NEXT:    s_sext_i32_i8 s4, s2
1503; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
1504; GFX9-DL-NEXT:    s_bfe_u32 s8, s2, 0x80010
1505; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 24
1506; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s9
1507; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 24
1508; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
1509; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s6, v2, v1
1510; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s4, v3, v1
1511; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s8, v4, v1
1512; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
1513; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s2, v2, v1
1514; GFX9-DL-NEXT:    global_store_short v0, v1, s[0:1]
1515; GFX9-DL-NEXT:    s_endpgm
1516;
1517; GFX10-DL-LABEL: notdot4_mixedtypes:
1518; GFX10-DL:       ; %bb.0: ; %entry
1519; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1520; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
1521; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1522; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1523; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1524; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
1525; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
1526; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
1527; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1528; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x80008
1529; GFX10-DL-NEXT:    s_bfe_u32 s3, s1, 0x80008
1530; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
1531; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s2, s3, v1
1532; GFX10-DL-NEXT:    s_sext_i32_i8 s2, s0
1533; GFX10-DL-NEXT:    s_sext_i32_i8 s3, s1
1534; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s2, s3, v1
1535; GFX10-DL-NEXT:    s_bfe_u32 s2, s0, 0x80010
1536; GFX10-DL-NEXT:    s_bfe_u32 s3, s1, 0x80010
1537; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 24
1538; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 24
1539; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s2, s3, v1
1540; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s0, s1, v1
1541; GFX10-DL-NEXT:    global_store_short v0, v1, s[4:5]
1542; GFX10-DL-NEXT:    s_endpgm
1543                                              <4 x i8> addrspace(1)* %src2,
1544                                              i16 addrspace(1)* nocapture %dst) {
1545entry:
1546  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
1547  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
1548
1549  %v1e0 = extractelement <4 x i8> %vec1, i64 0
1550  %cv1e0 = sext i8 %v1e0 to i16
1551  %v2e0 = extractelement <4 x i8> %vec2, i64 0
1552  %cv2e0 = sext i8 %v2e0 to i16
1553  %mul1 = mul nuw nsw i16 %cv1e0, %cv2e0
1554
1555  %v1e1 = extractelement <4 x i8> %vec1, i64 1
1556  %cv1e1 = zext i8 %v1e1 to i16
1557  %v2e1 = extractelement <4 x i8> %vec2, i64 1
1558  %cv2e1 = zext i8 %v2e1 to i16
1559  %mul2 = mul nuw nsw i16 %cv1e1, %cv2e1
1560
1561  %v1e2 = extractelement <4 x i8> %vec1, i64 2
1562  %cv1e2 = zext i8 %v1e2 to i16
1563  %v2e2 = extractelement <4 x i8> %vec2, i64 2
1564  %cv2e2 = zext i8 %v2e2 to i16
1565  %mul3 = mul nuw nsw i16 %cv1e2, %cv2e2
1566
1567  %v1e3 = extractelement <4 x i8> %vec1, i64 3
1568  %cv1e3 = zext i8 %v1e3 to i16
1569  %v2e3 = extractelement <4 x i8> %vec2, i64 3
1570  %cv2e3 = zext i8 %v2e3 to i16
1571  %mul4 = mul nuw nsw i16 %cv1e3, %cv2e3
1572
1573  %acc = load i16, i16 addrspace(1)* %dst, align 2
1574  %add1 = add i16 %mul2, %acc
1575  %add2 = add i16 %add1, %mul1
1576  %add3 = add i16 %add2, %mul3
1577  %add4 = add i16 %add3, %mul4
1578
1579  store i16 %add4, i16 addrspace(1)* %dst, align 2
1580  ret void
1581}
1582
1583; TODO: cleanup s_lshr_b32 and support this pattern.
1584define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
1585; GFX7-LABEL: udot4_acc32_vecMul:
1586; GFX7:       ; %bb.0: ; %entry
1587; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1588; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1589; GFX7-NEXT:    s_movk_i32 s11, 0xff
1590; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1591; GFX7-NEXT:    s_mov_b32 s2, -1
1592; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1593; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1594; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1595; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1596; GFX7-NEXT:    s_lshr_b32 s6, s4, 24
1597; GFX7-NEXT:    s_bfe_u32 s7, s4, 0x80008
1598; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x80010
1599; GFX7-NEXT:    s_lshr_b32 s8, s5, 24
1600; GFX7-NEXT:    s_bfe_u32 s9, s5, 0x80008
1601; GFX7-NEXT:    s_bfe_u32 s12, s5, 0x80010
1602; GFX7-NEXT:    s_and_b32 s5, s5, s11
1603; GFX7-NEXT:    s_and_b32 s4, s4, s11
1604; GFX7-NEXT:    s_load_dword s11, s[0:1], 0x0
1605; GFX7-NEXT:    v_mov_b32_e32 v0, s5
1606; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1607; GFX7-NEXT:    v_mov_b32_e32 v1, s11
1608; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v0, v1
1609; GFX7-NEXT:    v_mov_b32_e32 v1, s9
1610; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
1611; GFX7-NEXT:    v_mov_b32_e32 v1, s12
1612; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v1, v0
1613; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1614; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
1615; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1616; GFX7-NEXT:    s_endpgm
1617;
1618; GFX8-LABEL: udot4_acc32_vecMul:
1619; GFX8:       ; %bb.0: ; %entry
1620; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1621; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1622; GFX8-NEXT:    s_movk_i32 s2, 0xff
1623; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1624; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1625; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1626; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1627; GFX8-NEXT:    s_lshr_b32 s5, s3, 24
1628; GFX8-NEXT:    s_lshr_b32 s6, s4, 24
1629; GFX8-NEXT:    s_bfe_u32 s7, s3, 0x80010
1630; GFX8-NEXT:    v_lshrrev_b16_e64 v0, 8, s3
1631; GFX8-NEXT:    s_and_b32 s3, s3, s2
1632; GFX8-NEXT:    s_and_b32 s2, s4, s2
1633; GFX8-NEXT:    s_bfe_u32 s8, s4, 0x80010
1634; GFX8-NEXT:    v_lshrrev_b16_e64 v1, 8, s4
1635; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
1636; GFX8-NEXT:    v_mov_b32_e32 v2, s2
1637; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1638; GFX8-NEXT:    v_mov_b32_e32 v3, s4
1639; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v2, v3
1640; GFX8-NEXT:    v_mad_u32_u24 v0, v0, v1, v2
1641; GFX8-NEXT:    v_mov_b32_e32 v1, s8
1642; GFX8-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
1643; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1644; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v1, v0
1645; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1646; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1647; GFX8-NEXT:    flat_store_dword v[0:1], v2
1648; GFX8-NEXT:    s_endpgm
1649;
1650; GFX9-NODL-LABEL: udot4_acc32_vecMul:
1651; GFX9-NODL:       ; %bb.0: ; %entry
1652; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1653; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1654; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
1655; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
1656; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1657; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1658; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1659; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1660; GFX9-NODL-NEXT:    s_lshr_b32 s5, s3, 24
1661; GFX9-NODL-NEXT:    s_lshr_b32 s6, s4, 24
1662; GFX9-NODL-NEXT:    s_bfe_u32 s7, s3, 0x80010
1663; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
1664; GFX9-NODL-NEXT:    s_and_b32 s3, s3, s2
1665; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1666; GFX9-NODL-NEXT:    s_bfe_u32 s8, s4, 0x80010
1667; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v2, 8, s4
1668; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
1669; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
1670; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1671; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s4
1672; GFX9-NODL-NEXT:    v_mad_u32_u24 v3, s3, v3, v4
1673; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, v1, v2, v3
1674; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s8
1675; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s7, v2, v1
1676; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s6
1677; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s5, v2, v1
1678; GFX9-NODL-NEXT:    global_store_dword v0, v1, s[0:1]
1679; GFX9-NODL-NEXT:    s_endpgm
1680;
1681; GFX9-DL-LABEL: udot4_acc32_vecMul:
1682; GFX9-DL:       ; %bb.0: ; %entry
1683; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1684; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1685; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
1686; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
1687; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1688; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1689; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1690; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1691; GFX9-DL-NEXT:    s_lshr_b32 s5, s3, 24
1692; GFX9-DL-NEXT:    s_lshr_b32 s6, s4, 24
1693; GFX9-DL-NEXT:    s_bfe_u32 s7, s3, 0x80010
1694; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, s3
1695; GFX9-DL-NEXT:    s_and_b32 s3, s3, s2
1696; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
1697; GFX9-DL-NEXT:    s_bfe_u32 s8, s4, 0x80010
1698; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v2, 8, s4
1699; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
1700; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
1701; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1702; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
1703; GFX9-DL-NEXT:    v_mad_u32_u24 v3, s3, v3, v4
1704; GFX9-DL-NEXT:    v_mad_u32_u24 v1, v1, v2, v3
1705; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s8
1706; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s7, v2, v1
1707; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s6
1708; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s5, v2, v1
1709; GFX9-DL-NEXT:    global_store_dword v0, v1, s[0:1]
1710; GFX9-DL-NEXT:    s_endpgm
1711;
1712; GFX10-DL-LABEL: udot4_acc32_vecMul:
1713; GFX10-DL:       ; %bb.0: ; %entry
1714; GFX10-DL-NEXT:    s_clause 0x1
1715; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1716; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1717; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1718; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1719; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1720; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1721; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
1722; GFX10-DL-NEXT:    s_movk_i32 s6, 0xff
1723; GFX10-DL-NEXT:    s_mov_b32 s5, 0xffff
1724; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1725; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
1726; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s3
1727; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
1728; GFX10-DL-NEXT:    s_and_b32 s4, s2, s6
1729; GFX10-DL-NEXT:    s_and_b32 s6, s3, s6
1730; GFX10-DL-NEXT:    v_and_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
1731; GFX10-DL-NEXT:    v_and_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
1732; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s4, s6, v2
1733; GFX10-DL-NEXT:    s_bfe_u32 s4, s2, 0x80010
1734; GFX10-DL-NEXT:    s_bfe_u32 s5, s3, 0x80010
1735; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 24
1736; GFX10-DL-NEXT:    s_lshr_b32 s3, s3, 24
1737; GFX10-DL-NEXT:    v_mad_u32_u24 v0, v0, v1, v2
1738; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
1739; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s4, s5, v0
1740; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s2, s3, v0
1741; GFX10-DL-NEXT:    global_store_dword v1, v0, s[0:1]
1742; GFX10-DL-NEXT:    s_endpgm
1743                                              <4 x i8> addrspace(1)* %src2,
1744                                              i32 addrspace(1)* nocapture %dst) {
1745entry:
1746  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
1747  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
1748
1749  %cvec1 = zext <4 x i8> %vec1 to <4 x i32>
1750  %cvec2 = zext <4 x i8> %vec2 to <4 x i32>
1751
1752  %mul = mul <4 x i32> %cvec1, %cvec2
1753  %mul0 = extractelement <4 x i32> %mul, i64 0
1754  %mul1 = extractelement <4 x i32> %mul, i64 1
1755  %mul2 = extractelement <4 x i32> %mul, i64 2
1756  %mul3 = extractelement <4 x i32> %mul, i64 3
1757
1758  %acc = load i32, i32 addrspace(1)* %dst, align 4
1759  %add1 = add i32 %mul0, %acc
1760  %add2 = add i32 %add1, %mul1
1761  %add3 = add i32 %add2, %mul2
1762  %add4 = add i32 %add3, %mul3
1763
1764  store i32 %add4, i32 addrspace(1)* %dst, align 4
1765  ret void
1766}
1767
1768; TODO: This pattern should be recognized.
1769define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
1770; GFX7-LABEL: udot4_acc16_vecMul:
1771; GFX7:       ; %bb.0: ; %entry
1772; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1773; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1774; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1775; GFX7-NEXT:    s_mov_b32 s2, -1
1776; GFX7-NEXT:    s_movk_i32 s8, 0xff
1777; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1778; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
1779; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1780; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1781; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1782; GFX7-NEXT:    s_lshr_b32 s6, s4, 24
1783; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80008
1784; GFX7-NEXT:    s_bfe_u32 s12, s5, 0x80010
1785; GFX7-NEXT:    s_lshr_b32 s9, s5, 24
1786; GFX7-NEXT:    s_and_b32 s5, s5, s8
1787; GFX7-NEXT:    s_bfe_u32 s7, s4, 0x80008
1788; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
1789; GFX7-NEXT:    s_and_b32 s4, s4, s8
1790; GFX7-NEXT:    v_mov_b32_e32 v1, s5
1791; GFX7-NEXT:    v_mov_b32_e32 v2, s10
1792; GFX7-NEXT:    v_mov_b32_e32 v3, s12
1793; GFX7-NEXT:    s_waitcnt vmcnt(0)
1794; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1795; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v2, v0
1796; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
1797; GFX7-NEXT:    v_mov_b32_e32 v1, s9
1798; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
1799; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
1800; GFX7-NEXT:    s_endpgm
1801;
1802; GFX8-LABEL: udot4_acc16_vecMul:
1803; GFX8:       ; %bb.0: ; %entry
1804; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1805; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1806; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1807; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1808; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1809; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
1810; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
1811; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
1812; GFX8-NEXT:    s_movk_i32 s0, 0xff
1813; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1814; GFX8-NEXT:    s_and_b32 s6, s1, s0
1815; GFX8-NEXT:    s_and_b32 s0, s2, s0
1816; GFX8-NEXT:    v_mov_b32_e32 v5, s0
1817; GFX8-NEXT:    s_bfe_u32 s7, s2, 0x80010
1818; GFX8-NEXT:    v_lshrrev_b16_e64 v3, 8, s2
1819; GFX8-NEXT:    v_lshrrev_b16_e64 v4, 8, s1
1820; GFX8-NEXT:    s_lshr_b32 s4, s2, 24
1821; GFX8-NEXT:    s_bfe_u32 s5, s1, 0x80010
1822; GFX8-NEXT:    v_mov_b32_e32 v6, s7
1823; GFX8-NEXT:    s_lshr_b32 s3, s1, 24
1824; GFX8-NEXT:    s_waitcnt vmcnt(0)
1825; GFX8-NEXT:    v_mad_u32_u24 v2, s6, v5, v2
1826; GFX8-NEXT:    v_mad_u32_u24 v2, v4, v3, v2
1827; GFX8-NEXT:    v_mad_u32_u24 v2, s5, v6, v2
1828; GFX8-NEXT:    v_mov_b32_e32 v3, s4
1829; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
1830; GFX8-NEXT:    flat_store_short v[0:1], v2
1831; GFX8-NEXT:    s_endpgm
1832;
1833; GFX9-NODL-LABEL: udot4_acc16_vecMul:
1834; GFX9-NODL:       ; %bb.0: ; %entry
1835; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1836; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1837; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, 0xffff
1838; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
1839; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1840; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
1841; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
1842; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1843; GFX9-NODL-NEXT:    s_lshr_b32 s5, s2, 16
1844; GFX9-NODL-NEXT:    s_lshr_b32 s7, s3, 16
1845; GFX9-NODL-NEXT:    s_lshr_b32 s4, s2, 24
1846; GFX9-NODL-NEXT:    v_and_b32_sdwa v5, v3, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1847; GFX9-NODL-NEXT:    s_lshr_b32 s6, s3, 24
1848; GFX9-NODL-NEXT:    v_and_b32_sdwa v4, v3, s7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1849; GFX9-NODL-NEXT:    v_lshl_or_b32 v4, s6, 16, v4
1850; GFX9-NODL-NEXT:    v_lshl_or_b32 v5, s4, 16, v5
1851; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v4, v5, v4
1852; GFX9-NODL-NEXT:    v_and_b32_sdwa v5, v3, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1853; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v2, 8, s3
1854; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v1, 8, s2
1855; GFX9-NODL-NEXT:    v_and_b32_sdwa v3, v3, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1856; GFX9-NODL-NEXT:    v_lshl_or_b32 v2, v2, 16, v5
1857; GFX9-NODL-NEXT:    v_lshl_or_b32 v1, v1, 16, v3
1858; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v1, v1, v2
1859; GFX9-NODL-NEXT:    global_load_ushort v2, v0, s[0:1]
1860; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
1861; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v1, v2
1862; GFX9-NODL-NEXT:    v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1863; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v1, v4
1864; GFX9-NODL-NEXT:    v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1865; GFX9-NODL-NEXT:    global_store_short v0, v1, s[0:1]
1866; GFX9-NODL-NEXT:    s_endpgm
1867;
1868; GFX9-DL-LABEL: udot4_acc16_vecMul:
1869; GFX9-DL:       ; %bb.0: ; %entry
1870; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1871; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1872; GFX9-DL-NEXT:    v_mov_b32_e32 v3, 0xffff
1873; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
1874; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1875; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1876; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1877; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1878; GFX9-DL-NEXT:    s_lshr_b32 s5, s2, 16
1879; GFX9-DL-NEXT:    s_lshr_b32 s7, s3, 16
1880; GFX9-DL-NEXT:    s_lshr_b32 s4, s2, 24
1881; GFX9-DL-NEXT:    v_and_b32_sdwa v5, v3, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1882; GFX9-DL-NEXT:    s_lshr_b32 s6, s3, 24
1883; GFX9-DL-NEXT:    v_and_b32_sdwa v4, v3, s7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1884; GFX9-DL-NEXT:    v_lshl_or_b32 v4, s6, 16, v4
1885; GFX9-DL-NEXT:    v_lshl_or_b32 v5, s4, 16, v5
1886; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v4, v5, v4
1887; GFX9-DL-NEXT:    v_and_b32_sdwa v5, v3, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1888; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v2, 8, s3
1889; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, s2
1890; GFX9-DL-NEXT:    v_and_b32_sdwa v3, v3, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1891; GFX9-DL-NEXT:    v_lshl_or_b32 v2, v2, 16, v5
1892; GFX9-DL-NEXT:    v_lshl_or_b32 v1, v1, 16, v3
1893; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v1, v1, v2
1894; GFX9-DL-NEXT:    global_load_ushort v2, v0, s[0:1]
1895; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
1896; GFX9-DL-NEXT:    v_add_u32_e32 v2, v1, v2
1897; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1898; GFX9-DL-NEXT:    v_add_u32_e32 v1, v1, v4
1899; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1900; GFX9-DL-NEXT:    global_store_short v0, v1, s[0:1]
1901; GFX9-DL-NEXT:    s_endpgm
1902;
1903; GFX10-DL-LABEL: udot4_acc16_vecMul:
1904; GFX10-DL:       ; %bb.0: ; %entry
1905; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1906; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
1907; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1908; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
1909; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1910; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1911; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
1912; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
1913; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
1914; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1915; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, s0
1916; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1917; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v4, 8, s1
1918; GFX10-DL-NEXT:    v_and_b32_sdwa v5, v2, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1919; GFX10-DL-NEXT:    s_lshr_b32 s2, s1, 16
1920; GFX10-DL-NEXT:    s_lshr_b32 s3, s0, 16
1921; GFX10-DL-NEXT:    v_lshl_or_b32 v3, v3, 16, v6
1922; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 24
1923; GFX10-DL-NEXT:    v_lshl_or_b32 v4, v4, 16, v5
1924; GFX10-DL-NEXT:    v_and_b32_sdwa v5, v2, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1925; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v2, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1926; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 24
1927; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v4
1928; GFX10-DL-NEXT:    v_lshl_or_b32 v4, s1, 16, v5
1929; GFX10-DL-NEXT:    v_lshl_or_b32 v2, s0, 16, v2
1930; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v4
1931; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
1932; GFX10-DL-NEXT:    v_add_nc_u32_e32 v1, v3, v1
1933; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1934; GFX10-DL-NEXT:    v_add_nc_u32_e32 v1, v1, v2
1935; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1936; GFX10-DL-NEXT:    global_store_short v0, v1, s[4:5]
1937; GFX10-DL-NEXT:    s_endpgm
1938                                              <4 x i8> addrspace(1)* %src2,
1939                                              i16 addrspace(1)* nocapture %dst) {
1940entry:
1941  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
1942  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
1943
1944  %cvec1 = zext <4 x i8> %vec1 to <4 x i16>
1945  %cvec2 = zext <4 x i8> %vec2 to <4 x i16>
1946
1947  %mul = mul <4 x i16> %cvec1, %cvec2
1948  %mul0 = extractelement <4 x i16> %mul, i64 0
1949  %mul1 = extractelement <4 x i16> %mul, i64 1
1950  %mul2 = extractelement <4 x i16> %mul, i64 2
1951  %mul3 = extractelement <4 x i16> %mul, i64 3
1952
1953  %acc = load i16, i16 addrspace(1)* %dst, align 4
1954  %add1 = add i16 %mul0, %acc
1955  %add2 = add i16 %add1, %mul1
1956  %add3 = add i16 %add2, %mul2
1957  %add4 = add i16 %add3, %mul3
1958
1959  store i16 %add4, i16 addrspace(1)* %dst, align 4
1960  ret void
1961}
1962
1963; TODO: Support this pattern.
1964define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
1965; GFX7-LABEL: udot4_acc8_vecMul:
1966; GFX7:       ; %bb.0: ; %entry
1967; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1968; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1969; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1970; GFX7-NEXT:    s_mov_b32 s2, -1
1971; GFX7-NEXT:    s_movk_i32 s8, 0xff
1972; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1973; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
1974; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1975; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1976; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1977; GFX7-NEXT:    s_bfe_u32 s6, s4, 0x80008
1978; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80008
1979; GFX7-NEXT:    s_lshr_b32 s11, s5, 16
1980; GFX7-NEXT:    s_lshr_b32 s12, s5, 24
1981; GFX7-NEXT:    v_mov_b32_e32 v3, s10
1982; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
1983; GFX7-NEXT:    v_mov_b32_e32 v2, s11
1984; GFX7-NEXT:    s_lshr_b32 s9, s4, 24
1985; GFX7-NEXT:    v_mov_b32_e32 v1, s12
1986; GFX7-NEXT:    s_mul_i32 s4, s4, s5
1987; GFX7-NEXT:    v_mul_u32_u24_e32 v1, s9, v1
1988; GFX7-NEXT:    v_mul_u32_u24_e32 v2, s7, v2
1989; GFX7-NEXT:    v_mul_u32_u24_e32 v3, s6, v3
1990; GFX7-NEXT:    s_and_b32 s5, s4, s8
1991; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
1992; GFX7-NEXT:    v_and_b32_e32 v2, s8, v2
1993; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
1994; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
1995; GFX7-NEXT:    v_or_b32_e32 v2, s5, v3
1996; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
1997; GFX7-NEXT:    v_and_b32_e32 v2, 0xffff, v2
1998; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
1999; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 8, v1
2000; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
2001; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
2002; GFX7-NEXT:    s_waitcnt vmcnt(0)
2003; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
2004; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
2005; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
2006; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
2007; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
2008; GFX7-NEXT:    s_endpgm
2009;
2010; GFX8-LABEL: udot4_acc8_vecMul:
2011; GFX8:       ; %bb.0: ; %entry
2012; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2013; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2014; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2015; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2016; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2017; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
2018; GFX8-NEXT:    s_movk_i32 s0, 0xff
2019; GFX8-NEXT:    v_mov_b32_e32 v3, s0
2020; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
2021; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
2022; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2023; GFX8-NEXT:    s_lshr_b32 s2, s0, 24
2024; GFX8-NEXT:    s_lshr_b32 s4, s1, 24
2025; GFX8-NEXT:    s_lshr_b32 s3, s0, 16
2026; GFX8-NEXT:    v_mov_b32_e32 v4, s0
2027; GFX8-NEXT:    v_mov_b32_e32 v5, s1
2028; GFX8-NEXT:    s_mul_i32 s0, s0, s1
2029; GFX8-NEXT:    s_lshr_b32 s5, s1, 16
2030; GFX8-NEXT:    v_mul_u32_u24_sdwa v4, v4, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
2031; GFX8-NEXT:    v_mov_b32_e32 v5, s5
2032; GFX8-NEXT:    v_and_b32_e32 v3, s0, v3
2033; GFX8-NEXT:    v_mov_b32_e32 v6, s4
2034; GFX8-NEXT:    v_mov_b32_e32 v7, s2
2035; GFX8-NEXT:    v_or_b32_e32 v3, v3, v4
2036; GFX8-NEXT:    v_mul_u32_u24_e32 v5, s3, v5
2037; GFX8-NEXT:    v_mul_u32_u24_sdwa v6, v7, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2038; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff, v3
2039; GFX8-NEXT:    v_or_b32_sdwa v4, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2040; GFX8-NEXT:    v_or_b32_e32 v4, v3, v4
2041; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
2042; GFX8-NEXT:    s_waitcnt vmcnt(0)
2043; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
2044; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
2045; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2046; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2047; GFX8-NEXT:    flat_store_byte v[0:1], v2
2048; GFX8-NEXT:    s_endpgm
2049;
2050; GFX9-NODL-LABEL: udot4_acc8_vecMul:
2051; GFX9-NODL:       ; %bb.0: ; %entry
2052; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2053; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2054; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
2055; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2056; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
2057; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
2058; GFX9-NODL-NEXT:    global_load_ubyte v4, v0, s[0:1]
2059; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2060; GFX9-NODL-NEXT:    s_lshr_b32 s4, s2, 16
2061; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
2062; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s3
2063; GFX9-NODL-NEXT:    s_lshr_b32 s6, s3, 16
2064; GFX9-NODL-NEXT:    s_lshr_b32 s7, s3, 24
2065; GFX9-NODL-NEXT:    v_mul_lo_u16_e32 v1, s2, v1
2066; GFX9-NODL-NEXT:    v_mul_lo_u16_sdwa v2, s2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
2067; GFX9-NODL-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2068; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s6
2069; GFX9-NODL-NEXT:    s_lshr_b32 s5, s2, 24
2070; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s7
2071; GFX9-NODL-NEXT:    v_mul_lo_u16_sdwa v2, s5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2072; GFX9-NODL-NEXT:    v_mul_lo_u16_e32 v3, s4, v3
2073; GFX9-NODL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
2074; GFX9-NODL-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2075; GFX9-NODL-NEXT:    v_or_b32_e32 v2, v1, v2
2076; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v3, 8, v2
2077; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
2078; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v1, v4
2079; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v1, v3
2080; GFX9-NODL-NEXT:    v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2081; GFX9-NODL-NEXT:    v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2082; GFX9-NODL-NEXT:    global_store_byte v0, v1, s[0:1]
2083; GFX9-NODL-NEXT:    s_endpgm
2084;
2085; GFX9-DL-LABEL: udot4_acc8_vecMul:
2086; GFX9-DL:       ; %bb.0: ; %entry
2087; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2088; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2089; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
2090; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2091; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2092; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2093; GFX9-DL-NEXT:    global_load_ubyte v4, v0, s[0:1]
2094; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2095; GFX9-DL-NEXT:    s_lshr_b32 s4, s2, 16
2096; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
2097; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
2098; GFX9-DL-NEXT:    s_lshr_b32 s6, s3, 16
2099; GFX9-DL-NEXT:    s_lshr_b32 s7, s3, 24
2100; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v1, s2, v1
2101; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v2, s2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
2102; GFX9-DL-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2103; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s6
2104; GFX9-DL-NEXT:    s_lshr_b32 s5, s2, 24
2105; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s7
2106; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v2, s5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2107; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v3, s4, v3
2108; GFX9-DL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
2109; GFX9-DL-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2110; GFX9-DL-NEXT:    v_or_b32_e32 v2, v1, v2
2111; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v2
2112; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2113; GFX9-DL-NEXT:    v_add_u32_e32 v1, v1, v4
2114; GFX9-DL-NEXT:    v_add_u32_e32 v1, v1, v3
2115; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2116; GFX9-DL-NEXT:    v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2117; GFX9-DL-NEXT:    global_store_byte v0, v1, s[0:1]
2118; GFX9-DL-NEXT:    s_endpgm
2119;
2120; GFX10-DL-LABEL: udot4_acc8_vecMul:
2121; GFX10-DL:       ; %bb.0: ; %entry
2122; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
2123; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
2124; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2125; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2126; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2127; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
2128; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
2129; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
2130; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2131; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v2, 8, s0
2132; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, s1
2133; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 24
2134; GFX10-DL-NEXT:    s_lshr_b32 s3, s1, 24
2135; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, s2, s3
2136; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v2, v2, v3
2137; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v3, s0, s1
2138; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 16
2139; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 16
2140; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v2, 8, v2
2141; GFX10-DL-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2142; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v3, 8, v4
2143; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, s0, s1
2144; GFX10-DL-NEXT:    v_and_b32_e32 v2, 0xffff, v2
2145; GFX10-DL-NEXT:    v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2146; GFX10-DL-NEXT:    v_or_b32_e32 v3, v2, v3
2147; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v4, 8, v3
2148; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
2149; GFX10-DL-NEXT:    v_add_nc_u32_e32 v1, v2, v1
2150; GFX10-DL-NEXT:    v_add_nc_u32_e32 v1, v1, v4
2151; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2152; GFX10-DL-NEXT:    v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
2153; GFX10-DL-NEXT:    global_store_byte v0, v1, s[4:5]
2154; GFX10-DL-NEXT:    s_endpgm
2155                                             <4 x i8> addrspace(1)* %src2,
2156                                             i8 addrspace(1)* nocapture %dst) {
2157entry:
2158  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
2159  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
2160
2161  %mul = mul <4 x i8> %vec1, %vec2
2162  %mul0 = extractelement <4 x i8> %mul, i64 0
2163  %mul1 = extractelement <4 x i8> %mul, i64 1
2164  %mul2 = extractelement <4 x i8> %mul, i64 2
2165  %mul3 = extractelement <4 x i8> %mul, i64 3
2166
2167  %acc = load i8, i8 addrspace(1)* %dst, align 4
2168  %add1 = add i8 %mul0, %acc
2169  %add2 = add i8 %add1, %mul1
2170  %add3 = add i8 %add2, %mul2
2171  %add4 = add i8 %add3, %mul3
2172
2173  store i8 %add4, i8 addrspace(1)* %dst, align 4
2174  ret void
2175}
2176