1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-NODL %s
5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
8
9define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
10; GFX7-LABEL: idot4_acc32:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
13; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
14; GFX7-NEXT:    s_mov_b32 s3, 0xf000
15; GFX7-NEXT:    s_mov_b32 s10, 0
16; GFX7-NEXT:    s_mov_b32 s11, s3
17; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
18; GFX7-NEXT:    s_mov_b64 s[8:9], s[4:5]
19; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
20; GFX7-NEXT:    v_mov_b32_e32 v1, 0
21; GFX7-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
22; GFX7-NEXT:    s_mov_b64 s[8:9], s[6:7]
23; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
24; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
25; GFX7-NEXT:    s_mov_b32 s2, -1
26; GFX7-NEXT:    s_waitcnt vmcnt(1)
27; GFX7-NEXT:    v_bfe_i32 v1, v2, 0, 8
28; GFX7-NEXT:    v_bfe_i32 v3, v2, 8, 8
29; GFX7-NEXT:    s_waitcnt vmcnt(0)
30; GFX7-NEXT:    v_bfe_i32 v5, v0, 0, 8
31; GFX7-NEXT:    v_bfe_i32 v6, v0, 8, 8
32; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
33; GFX7-NEXT:    v_mad_i32_i24 v1, v1, v5, s4
34; GFX7-NEXT:    v_bfe_i32 v4, v2, 16, 8
35; GFX7-NEXT:    v_bfe_i32 v7, v0, 16, 8
36; GFX7-NEXT:    v_mad_i32_i24 v1, v3, v6, v1
37; GFX7-NEXT:    v_ashrrev_i32_e32 v2, 24, v2
38; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
39; GFX7-NEXT:    v_mad_i32_i24 v1, v4, v7, v1
40; GFX7-NEXT:    v_mad_i32_i24 v0, v2, v0, v1
41; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
42; GFX7-NEXT:    s_endpgm
43;
44; GFX8-LABEL: idot4_acc32:
45; GFX8:       ; %bb.0: ; %entry
46; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
47; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
48; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
49; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
50; GFX8-NEXT:    v_mov_b32_e32 v1, s5
51; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
52; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
53; GFX8-NEXT:    flat_load_dword v3, v[0:1]
54; GFX8-NEXT:    v_mov_b32_e32 v1, s7
55; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
56; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
57; GFX8-NEXT:    flat_load_dword v0, v[0:1]
58; GFX8-NEXT:    s_load_dword s2, s[0:1], 0x0
59; GFX8-NEXT:    s_waitcnt vmcnt(1)
60; GFX8-NEXT:    v_bfe_i32 v1, v3, 0, 8
61; GFX8-NEXT:    v_bfe_i32 v4, v3, 8, 8
62; GFX8-NEXT:    v_bfe_i32 v6, v3, 16, 8
63; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 24, v3
64; GFX8-NEXT:    s_waitcnt vmcnt(0)
65; GFX8-NEXT:    v_bfe_i32 v2, v0, 0, 8
66; GFX8-NEXT:    v_bfe_i32 v5, v0, 8, 8
67; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
68; GFX8-NEXT:    v_mad_i32_i24 v1, v1, v2, s2
69; GFX8-NEXT:    v_bfe_i32 v7, v0, 16, 8
70; GFX8-NEXT:    v_mad_i32_i24 v1, v4, v5, v1
71; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
72; GFX8-NEXT:    v_mad_i32_i24 v1, v6, v7, v1
73; GFX8-NEXT:    v_mad_i32_i24 v2, v3, v0, v1
74; GFX8-NEXT:    v_mov_b32_e32 v0, s0
75; GFX8-NEXT:    v_mov_b32_e32 v1, s1
76; GFX8-NEXT:    flat_store_dword v[0:1], v2
77; GFX8-NEXT:    s_endpgm
78;
79; GFX9-NODL-LABEL: idot4_acc32:
80; GFX9-NODL:       ; %bb.0: ; %entry
81; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
82; GFX9-NODL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
83; GFX9-NODL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
84; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
85; GFX9-NODL-NEXT:    global_load_dword v1, v0, s[4:5]
86; GFX9-NODL-NEXT:    global_load_dword v2, v0, s[6:7]
87; GFX9-NODL-NEXT:    s_load_dword s0, s[2:3], 0x0
88; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
89; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
90; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
91; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v4, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
92; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
93; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
94; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
95; GFX9-NODL-NEXT:    v_add3_u32 v2, v3, s0, v4
96; GFX9-NODL-NEXT:    v_add3_u32 v1, v2, v5, v1
97; GFX9-NODL-NEXT:    global_store_dword v0, v1, s[2:3]
98; GFX9-NODL-NEXT:    s_endpgm
99;
100; GFX9-DL-LABEL: idot4_acc32:
101; GFX9-DL:       ; %bb.0: ; %entry
102; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
103; GFX9-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
104; GFX9-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
105; GFX9-DL-NEXT:    v_mov_b32_e32 v1, 0
106; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
107; GFX9-DL-NEXT:    global_load_dword v2, v0, s[4:5]
108; GFX9-DL-NEXT:    global_load_dword v3, v0, s[6:7]
109; GFX9-DL-NEXT:    s_load_dword s0, s[2:3], 0x0
110; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
111; GFX9-DL-NEXT:    v_dot4_i32_i8 v0, v2, v3, s0
112; GFX9-DL-NEXT:    global_store_dword v1, v0, s[2:3]
113; GFX9-DL-NEXT:    s_endpgm
114;
115; GFX10-DL-LABEL: idot4_acc32:
116; GFX10-DL:       ; %bb.0: ; %entry
117; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
118; GFX10-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
119; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
120; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
121; GFX10-DL-NEXT:    s_clause 0x1
122; GFX10-DL-NEXT:    global_load_dword v1, v0, s[4:5]
123; GFX10-DL-NEXT:    global_load_dword v2, v0, s[6:7]
124; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
125; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
126; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
127; GFX10-DL-NEXT:    v_dot4_i32_i8 v1, v1, v2, s2
128; GFX10-DL-NEXT:    global_store_dword v0, v1, s[0:1]
129; GFX10-DL-NEXT:    s_endpgm
130                                       <4 x i8> addrspace(1)* %src2,
131                                       i32 addrspace(1)* nocapture %dst) {
132entry:
133  %idx = call i32 @llvm.amdgcn.workitem.id.x()
134  %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
135  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
136  %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
137  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
138
139  %v1e0 = extractelement <4 x i8> %vec1, i64 0
140  %cv1e0 = sext i8 %v1e0 to i32
141  %v2e0 = extractelement <4 x i8> %vec2, i64 0
142  %cv2e0 = sext i8 %v2e0 to i32
143  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
144
145  %v1e1 = extractelement <4 x i8> %vec1, i64 1
146  %cv1e1 = sext i8 %v1e1 to i32
147  %v2e1 = extractelement <4 x i8> %vec2, i64 1
148  %cv2e1 = sext i8 %v2e1 to i32
149  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
150
151  %v1e2 = extractelement <4 x i8> %vec1, i64 2
152  %cv1e2 = sext i8 %v1e2 to i32
153  %v2e2 = extractelement <4 x i8> %vec2, i64 2
154  %cv2e2 = sext i8 %v2e2 to i32
155  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
156
157  %v1e3 = extractelement <4 x i8> %vec1, i64 3
158  %cv1e3 = sext i8 %v1e3 to i32
159  %v2e3 = extractelement <4 x i8> %vec2, i64 3
160  %cv2e3 = sext i8 %v2e3 to i32
161  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
162
163  %acc = load i32, i32 addrspace(1)* %dst, align 4
164  %add1 = add i32 %mul1, %acc
165  %add2 = add i32 %add1, %mul2
166  %add3 = add i32 %add2, %mul3
167  %add4 = add i32 %add3, %mul4
168  store i32 %add4, i32 addrspace(1)* %dst, align 4
169  ret void
170}
171
172; TODO: Currently, vector elements{0 and 3} get zero_extended from i16 to i32 which should
173; be sign_extended directly to i32; prevents the pattern recognizer to recognize this pattern.
174define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
175; GFX7-LABEL: idot4_acc16:
176; GFX7:       ; %bb.0: ; %entry
177; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
178; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
179; GFX7-NEXT:    s_mov_b32 s3, 0xf000
180; GFX7-NEXT:    s_mov_b32 s10, 0
181; GFX7-NEXT:    s_mov_b32 s11, s3
182; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
183; GFX7-NEXT:    s_mov_b64 s[8:9], s[4:5]
184; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
185; GFX7-NEXT:    v_mov_b32_e32 v1, 0
186; GFX7-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
187; GFX7-NEXT:    s_mov_b64 s[8:9], s[6:7]
188; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
189; GFX7-NEXT:    s_mov_b32 s2, -1
190; GFX7-NEXT:    buffer_load_ushort v8, off, s[0:3], 0
191; GFX7-NEXT:    s_mov_b32 s4, 0xffff
192; GFX7-NEXT:    s_waitcnt vmcnt(2)
193; GFX7-NEXT:    v_bfe_i32 v1, v2, 0, 8
194; GFX7-NEXT:    v_bfe_i32 v3, v2, 8, 8
195; GFX7-NEXT:    s_waitcnt vmcnt(1)
196; GFX7-NEXT:    v_bfe_i32 v5, v0, 0, 8
197; GFX7-NEXT:    v_bfe_i32 v6, v0, 8, 8
198; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
199; GFX7-NEXT:    v_and_b32_e32 v5, s4, v5
200; GFX7-NEXT:    v_bfe_i32 v4, v2, 16, 8
201; GFX7-NEXT:    v_bfe_i32 v7, v0, 16, 8
202; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
203; GFX7-NEXT:    v_and_b32_e32 v6, s4, v6
204; GFX7-NEXT:    s_waitcnt vmcnt(0)
205; GFX7-NEXT:    v_mad_u32_u24 v1, v1, v5, v8
206; GFX7-NEXT:    v_ashrrev_i32_e32 v2, 24, v2
207; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
208; GFX7-NEXT:    v_and_b32_e32 v4, s4, v4
209; GFX7-NEXT:    v_and_b32_e32 v7, s4, v7
210; GFX7-NEXT:    v_mad_u32_u24 v1, v3, v6, v1
211; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
212; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
213; GFX7-NEXT:    v_mad_u32_u24 v1, v4, v7, v1
214; GFX7-NEXT:    v_mad_u32_u24 v0, v2, v0, v1
215; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
216; GFX7-NEXT:    s_endpgm
217;
218; GFX8-LABEL: idot4_acc16:
219; GFX8:       ; %bb.0: ; %entry
220; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
221; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
222; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
223; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
224; GFX8-NEXT:    v_mov_b32_e32 v1, s5
225; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
226; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
227; GFX8-NEXT:    flat_load_dword v4, v[0:1]
228; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
229; GFX8-NEXT:    v_mov_b32_e32 v1, s7
230; GFX8-NEXT:    v_mov_b32_e32 v3, s1
231; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
232; GFX8-NEXT:    v_mov_b32_e32 v2, s0
233; GFX8-NEXT:    flat_load_dword v0, v[0:1]
234; GFX8-NEXT:    flat_load_ushort v1, v[2:3]
235; GFX8-NEXT:    s_waitcnt vmcnt(2)
236; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 8, v4
237; GFX8-NEXT:    v_bfe_i32 v7, v4, 0, 8
238; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
239; GFX8-NEXT:    v_bfe_i32 v9, v9, 0, 8
240; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 24, v4
241; GFX8-NEXT:    s_waitcnt vmcnt(1)
242; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 8, v0
243; GFX8-NEXT:    v_bfe_i32 v8, v0, 0, 8
244; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
245; GFX8-NEXT:    v_bfe_i32 v10, v10, 0, 8
246; GFX8-NEXT:    s_waitcnt vmcnt(0)
247; GFX8-NEXT:    v_mad_u16 v1, v7, v8, v1
248; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
249; GFX8-NEXT:    v_bfe_i32 v5, v5, 0, 8
250; GFX8-NEXT:    v_bfe_i32 v6, v6, 0, 8
251; GFX8-NEXT:    v_mad_u16 v1, v9, v10, v1
252; GFX8-NEXT:    v_bfe_i32 v4, v4, 0, 8
253; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 8
254; GFX8-NEXT:    v_mad_u16 v1, v5, v6, v1
255; GFX8-NEXT:    v_mad_u16 v0, v4, v0, v1
256; GFX8-NEXT:    flat_store_short v[2:3], v0
257; GFX8-NEXT:    s_endpgm
258;
259; GFX9-NODL-LABEL: idot4_acc16:
260; GFX9-NODL:       ; %bb.0: ; %entry
261; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
262; GFX9-NODL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
263; GFX9-NODL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
264; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, 0
265; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
266; GFX9-NODL-NEXT:    global_load_dword v2, v0, s[4:5]
267; GFX9-NODL-NEXT:    global_load_dword v3, v0, s[6:7]
268; GFX9-NODL-NEXT:    global_load_ushort v4, v1, s[2:3]
269; GFX9-NODL-NEXT:    s_waitcnt vmcnt(2)
270; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v8, 8, v2
271; GFX9-NODL-NEXT:    s_waitcnt vmcnt(1)
272; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v9, 8, v3
273; GFX9-NODL-NEXT:    v_bfe_i32 v6, v2, 0, 8
274; GFX9-NODL-NEXT:    v_bfe_i32 v7, v3, 0, 8
275; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
276; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
277; GFX9-NODL-NEXT:    v_bfe_i32 v8, v8, 0, 8
278; GFX9-NODL-NEXT:    v_bfe_i32 v9, v9, 0, 8
279; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
280; GFX9-NODL-NEXT:    v_mad_legacy_u16 v4, v6, v7, v4
281; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
282; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v3, 24, v3
283; GFX9-NODL-NEXT:    v_bfe_i32 v0, v0, 0, 8
284; GFX9-NODL-NEXT:    v_bfe_i32 v5, v5, 0, 8
285; GFX9-NODL-NEXT:    v_mad_legacy_u16 v4, v8, v9, v4
286; GFX9-NODL-NEXT:    v_bfe_i32 v2, v2, 0, 8
287; GFX9-NODL-NEXT:    v_bfe_i32 v3, v3, 0, 8
288; GFX9-NODL-NEXT:    v_mad_legacy_u16 v0, v0, v5, v4
289; GFX9-NODL-NEXT:    v_mad_legacy_u16 v0, v2, v3, v0
290; GFX9-NODL-NEXT:    global_store_short v1, v0, s[2:3]
291; GFX9-NODL-NEXT:    s_endpgm
292;
293; GFX9-DL-LABEL: idot4_acc16:
294; GFX9-DL:       ; %bb.0: ; %entry
295; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
296; GFX9-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
297; GFX9-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
298; GFX9-DL-NEXT:    v_mov_b32_e32 v1, 0
299; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
300; GFX9-DL-NEXT:    global_load_dword v2, v0, s[4:5]
301; GFX9-DL-NEXT:    global_load_dword v3, v0, s[6:7]
302; GFX9-DL-NEXT:    global_load_ushort v4, v1, s[2:3]
303; GFX9-DL-NEXT:    s_waitcnt vmcnt(2)
304; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v8, 8, v2
305; GFX9-DL-NEXT:    s_waitcnt vmcnt(1)
306; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v9, 8, v3
307; GFX9-DL-NEXT:    v_bfe_i32 v6, v2, 0, 8
308; GFX9-DL-NEXT:    v_bfe_i32 v7, v3, 0, 8
309; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
310; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
311; GFX9-DL-NEXT:    v_bfe_i32 v8, v8, 0, 8
312; GFX9-DL-NEXT:    v_bfe_i32 v9, v9, 0, 8
313; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
314; GFX9-DL-NEXT:    v_mad_legacy_u16 v4, v6, v7, v4
315; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
316; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v3, 24, v3
317; GFX9-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
318; GFX9-DL-NEXT:    v_bfe_i32 v5, v5, 0, 8
319; GFX9-DL-NEXT:    v_mad_legacy_u16 v4, v8, v9, v4
320; GFX9-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
321; GFX9-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
322; GFX9-DL-NEXT:    v_mad_legacy_u16 v0, v0, v5, v4
323; GFX9-DL-NEXT:    v_mad_legacy_u16 v0, v2, v3, v0
324; GFX9-DL-NEXT:    global_store_short v1, v0, s[2:3]
325; GFX9-DL-NEXT:    s_endpgm
326;
327; GFX10-DL-LABEL: idot4_acc16:
328; GFX10-DL:       ; %bb.0: ; %entry
329; GFX10-DL-NEXT:    s_clause 0x1
330; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
331; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
332; GFX10-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
333; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
334; GFX10-DL-NEXT:    s_clause 0x1
335; GFX10-DL-NEXT:    global_load_dword v1, v0, s[4:5]
336; GFX10-DL-NEXT:    global_load_dword v2, v0, s[6:7]
337; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
338; GFX10-DL-NEXT:    global_load_ushort v3, v0, s[2:3]
339; GFX10-DL-NEXT:    s_waitcnt vmcnt(2)
340; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
341; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
342; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v6, 8, v2
343; GFX10-DL-NEXT:    v_bfe_i32 v4, v1, 0, 8
344; GFX10-DL-NEXT:    v_bfe_i32 v7, v2, 0, 8
345; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v8, 16, v1
346; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v9, 16, v2
347; GFX10-DL-NEXT:    v_bfe_i32 v5, v5, 0, 8
348; GFX10-DL-NEXT:    v_bfe_i32 v6, v6, 0, 8
349; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
350; GFX10-DL-NEXT:    v_mad_u16 v3, v4, v7, v3
351; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
352; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
353; GFX10-DL-NEXT:    v_bfe_i32 v4, v8, 0, 8
354; GFX10-DL-NEXT:    v_bfe_i32 v7, v9, 0, 8
355; GFX10-DL-NEXT:    v_mad_u16 v3, v5, v6, v3
356; GFX10-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
357; GFX10-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
358; GFX10-DL-NEXT:    v_mad_u16 v3, v4, v7, v3
359; GFX10-DL-NEXT:    v_mad_u16 v1, v1, v2, v3
360; GFX10-DL-NEXT:    global_store_short v0, v1, s[2:3]
361; GFX10-DL-NEXT:    s_endpgm
362                                       <4 x i8> addrspace(1)* %src2,
363                                       i16 addrspace(1)* nocapture %dst) {
364entry:
365  %idx = call i32 @llvm.amdgcn.workitem.id.x()
366  %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
367  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
368  %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
369  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
370
371  %v1e0 = extractelement <4 x i8> %vec1, i64 0
372  %cv1e0 = sext i8 %v1e0 to i16
373  %v2e0 = extractelement <4 x i8> %vec2, i64 0
374  %cv2e0 = sext i8 %v2e0 to i16
375  %mul1 = mul nsw i16 %cv1e0, %cv2e0
376
377  %v1e1 = extractelement <4 x i8> %vec1, i64 1
378  %cv1e1 = sext i8 %v1e1 to i16
379  %v2e1 = extractelement <4 x i8> %vec2, i64 1
380  %cv2e1 = sext i8 %v2e1 to i16
381  %mul2 = mul nsw i16 %cv1e1, %cv2e1
382
383  %v1e2 = extractelement <4 x i8> %vec1, i64 2
384  %cv1e2 = sext i8 %v1e2 to i16
385  %v2e2 = extractelement <4 x i8> %vec2, i64 2
386  %cv2e2 = sext i8 %v2e2 to i16
387  %mul3 = mul nsw i16 %cv1e2, %cv2e2
388
389  %v1e3 = extractelement <4 x i8> %vec1, i64 3
390  %cv1e3 = sext i8 %v1e3 to i16
391  %v2e3 = extractelement <4 x i8> %vec2, i64 3
392  %cv2e3 = sext i8 %v2e3 to i16
393  %mul4 = mul nsw i16 %cv1e3, %cv2e3
394
395  %acc = load i16, i16 addrspace(1)* %dst, align 2
396  %add1 = add i16 %mul1, %acc
397  %add2 = add i16 %add1, %mul2
398  %add3 = add i16 %add2, %mul3
399  %add4 = add i16 %add3, %mul4
400  store i16 %add4, i16 addrspace(1)* %dst, align 2
401  ret void
402}
403
404define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
405; GFX7-LABEL: idot4_acc8:
406; GFX7:       ; %bb.0: ; %entry
407; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
408; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
409; GFX7-NEXT:    s_mov_b32 s3, 0xf000
410; GFX7-NEXT:    s_mov_b32 s10, 0
411; GFX7-NEXT:    s_mov_b32 s11, s3
412; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
413; GFX7-NEXT:    s_mov_b64 s[8:9], s[4:5]
414; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
415; GFX7-NEXT:    v_mov_b32_e32 v1, 0
416; GFX7-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
417; GFX7-NEXT:    s_mov_b64 s[8:9], s[6:7]
418; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
419; GFX7-NEXT:    s_mov_b32 s2, -1
420; GFX7-NEXT:    buffer_load_ubyte v8, off, s[0:3], 0
421; GFX7-NEXT:    s_movk_i32 s4, 0xff
422; GFX7-NEXT:    s_waitcnt vmcnt(2)
423; GFX7-NEXT:    v_and_b32_e32 v1, s4, v2
424; GFX7-NEXT:    v_bfe_u32 v3, v2, 8, 8
425; GFX7-NEXT:    s_waitcnt vmcnt(1)
426; GFX7-NEXT:    v_and_b32_e32 v5, s4, v0
427; GFX7-NEXT:    v_bfe_u32 v6, v0, 8, 8
428; GFX7-NEXT:    s_waitcnt vmcnt(0)
429; GFX7-NEXT:    v_mad_u32_u24 v1, v1, v5, v8
430; GFX7-NEXT:    v_bfe_u32 v4, v2, 16, 8
431; GFX7-NEXT:    v_bfe_u32 v7, v0, 16, 8
432; GFX7-NEXT:    v_mad_u32_u24 v1, v3, v6, v1
433; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
434; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
435; GFX7-NEXT:    v_mad_u32_u24 v1, v4, v7, v1
436; GFX7-NEXT:    v_mad_u32_u24 v0, v2, v0, v1
437; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
438; GFX7-NEXT:    s_endpgm
439;
440; GFX8-LABEL: idot4_acc8:
441; GFX8:       ; %bb.0: ; %entry
442; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
443; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
444; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
445; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
446; GFX8-NEXT:    v_mov_b32_e32 v1, s5
447; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
448; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
449; GFX8-NEXT:    flat_load_dword v4, v[0:1]
450; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
451; GFX8-NEXT:    v_mov_b32_e32 v3, s1
452; GFX8-NEXT:    v_mov_b32_e32 v1, s7
453; GFX8-NEXT:    v_mov_b32_e32 v2, s0
454; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
455; GFX8-NEXT:    flat_load_ubyte v5, v[2:3]
456; GFX8-NEXT:    flat_load_dword v0, v[0:1]
457; GFX8-NEXT:    s_waitcnt vmcnt(2)
458; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 8, v4
459; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v4
460; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 24, v4
461; GFX8-NEXT:    s_waitcnt vmcnt(0)
462; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
463; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 8, v0
464; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 24, v0
465; GFX8-NEXT:    v_mad_u16 v0, v4, v0, v5
466; GFX8-NEXT:    v_mad_u16 v0, v7, v8, v0
467; GFX8-NEXT:    v_mad_u16 v0, v1, v6, v0
468; GFX8-NEXT:    v_mad_u16 v0, v9, v10, v0
469; GFX8-NEXT:    flat_store_byte v[2:3], v0
470; GFX8-NEXT:    s_endpgm
471;
472; GFX9-NODL-LABEL: idot4_acc8:
473; GFX9-NODL:       ; %bb.0: ; %entry
474; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
475; GFX9-NODL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
476; GFX9-NODL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
477; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, 0
478; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
479; GFX9-NODL-NEXT:    global_load_dword v2, v0, s[4:5]
480; GFX9-NODL-NEXT:    global_load_dword v3, v0, s[6:7]
481; GFX9-NODL-NEXT:    global_load_ubyte v4, v1, s[2:3]
482; GFX9-NODL-NEXT:    s_waitcnt vmcnt(2)
483; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
484; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v6, 8, v2
485; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v8, 24, v2
486; GFX9-NODL-NEXT:    s_waitcnt vmcnt(1)
487; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v7, 8, v3
488; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
489; GFX9-NODL-NEXT:    v_mad_legacy_u16 v2, v2, v3, v4
490; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
491; GFX9-NODL-NEXT:    v_mad_legacy_u16 v2, v6, v7, v2
492; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v9, 24, v3
493; GFX9-NODL-NEXT:    v_mad_legacy_u16 v0, v0, v5, v2
494; GFX9-NODL-NEXT:    v_mad_legacy_u16 v0, v8, v9, v0
495; GFX9-NODL-NEXT:    global_store_byte v1, v0, s[2:3]
496; GFX9-NODL-NEXT:    s_endpgm
497;
498; GFX9-DL-LABEL: idot4_acc8:
499; GFX9-DL:       ; %bb.0: ; %entry
500; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
501; GFX9-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
502; GFX9-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
503; GFX9-DL-NEXT:    v_mov_b32_e32 v1, 0
504; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
505; GFX9-DL-NEXT:    global_load_dword v2, v0, s[4:5]
506; GFX9-DL-NEXT:    global_load_dword v3, v0, s[6:7]
507; GFX9-DL-NEXT:    global_load_ubyte v4, v1, s[2:3]
508; GFX9-DL-NEXT:    s_waitcnt vmcnt(2)
509; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
510; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v6, 8, v2
511; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v8, 24, v2
512; GFX9-DL-NEXT:    s_waitcnt vmcnt(1)
513; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v7, 8, v3
514; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
515; GFX9-DL-NEXT:    v_mad_legacy_u16 v2, v2, v3, v4
516; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
517; GFX9-DL-NEXT:    v_mad_legacy_u16 v2, v6, v7, v2
518; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v9, 24, v3
519; GFX9-DL-NEXT:    v_mad_legacy_u16 v0, v0, v5, v2
520; GFX9-DL-NEXT:    v_mad_legacy_u16 v0, v8, v9, v0
521; GFX9-DL-NEXT:    global_store_byte v1, v0, s[2:3]
522; GFX9-DL-NEXT:    s_endpgm
523;
524; GFX10-DL-LABEL: idot4_acc8:
525; GFX10-DL:       ; %bb.0: ; %entry
526; GFX10-DL-NEXT:    s_clause 0x1
527; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
528; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
529; GFX10-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
530; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
531; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
532; GFX10-DL-NEXT:    s_clause 0x1
533; GFX10-DL-NEXT:    global_load_dword v2, v0, s[4:5]
534; GFX10-DL-NEXT:    global_load_dword v3, v0, s[6:7]
535; GFX10-DL-NEXT:    global_load_ubyte v4, v1, s[2:3]
536; GFX10-DL-NEXT:    s_waitcnt vmcnt(2)
537; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
538; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
539; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
540; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
541; GFX10-DL-NEXT:    v_mad_u16 v4, v2, v3, v4
542; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
543; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
544; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
545; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v3, 24, v3
546; GFX10-DL-NEXT:    v_mad_u16 v0, v0, v5, v4
547; GFX10-DL-NEXT:    v_mad_u16 v0, v6, v7, v0
548; GFX10-DL-NEXT:    v_mad_u16 v0, v2, v3, v0
549; GFX10-DL-NEXT:    global_store_byte v1, v0, s[2:3]
550; GFX10-DL-NEXT:    s_endpgm
551                                      <4 x i8> addrspace(1)* %src2,
552                                      i8 addrspace(1)* nocapture %dst) {
553entry:
554  %idx = call i32 @llvm.amdgcn.workitem.id.x()
555  %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
556  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
557  %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
558  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
559
560  %v1e0 = extractelement <4 x i8> %vec1, i64 0
561  %v2e0 = extractelement <4 x i8> %vec2, i64 0
562  %mul1 = mul i8 %v1e0, %v2e0
563
564  %v1e1 = extractelement <4 x i8> %vec1, i64 1
565  %v2e1 = extractelement <4 x i8> %vec2, i64 1
566  %mul2 = mul i8 %v1e1, %v2e1
567
568  %v1e2 = extractelement <4 x i8> %vec1, i64 2
569  %v2e2 = extractelement <4 x i8> %vec2, i64 2
570  %mul3 = mul i8 %v1e2, %v2e2
571
572  %v1e3 = extractelement <4 x i8> %vec1, i64 3
573  %v2e3 = extractelement <4 x i8> %vec2, i64 3
574  %mul4 = mul i8 %v1e3, %v2e3
575
576  %acc = load i8, i8 addrspace(1)* %dst, align 2
577  %add1 = add i8 %mul1, %acc
578  %add2 = add i8 %add1, %mul2
579  %add3 = add i8 %add2, %mul3
580  %add4 = add nsw i8 %add3, %mul4
581  store i8 %add4, i8 addrspace(1)* %dst, align 2
582  ret void
583}
584
585define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
586; GFX7-LABEL: idot4_multiuse_mul1:
587; GFX7:       ; %bb.0: ; %entry
588; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
589; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
590; GFX7-NEXT:    s_mov_b32 s3, 0xf000
591; GFX7-NEXT:    s_mov_b32 s10, 0
592; GFX7-NEXT:    s_mov_b32 s11, s3
593; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
594; GFX7-NEXT:    s_mov_b64 s[8:9], s[4:5]
595; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
596; GFX7-NEXT:    v_mov_b32_e32 v1, 0
597; GFX7-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
598; GFX7-NEXT:    s_mov_b64 s[8:9], s[6:7]
599; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
600; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
601; GFX7-NEXT:    s_mov_b32 s2, -1
602; GFX7-NEXT:    s_waitcnt vmcnt(1)
603; GFX7-NEXT:    v_bfe_i32 v1, v2, 0, 8
604; GFX7-NEXT:    v_bfe_i32 v3, v2, 8, 8
605; GFX7-NEXT:    s_waitcnt vmcnt(0)
606; GFX7-NEXT:    v_bfe_i32 v5, v0, 0, 8
607; GFX7-NEXT:    v_bfe_i32 v6, v0, 8, 8
608; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
609; GFX7-NEXT:    v_mad_i32_i24 v8, v1, v5, s4
610; GFX7-NEXT:    v_mad_i32_i24 v3, v3, v6, v8
611; GFX7-NEXT:    v_bfe_i32 v4, v2, 16, 8
612; GFX7-NEXT:    v_bfe_i32 v7, v0, 16, 8
613; GFX7-NEXT:    v_mad_i32_i24 v1, v1, v5, v3
614; GFX7-NEXT:    v_ashrrev_i32_e32 v2, 24, v2
615; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
616; GFX7-NEXT:    v_mad_i32_i24 v1, v4, v7, v1
617; GFX7-NEXT:    v_mad_i32_i24 v0, v2, v0, v1
618; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
619; GFX7-NEXT:    s_endpgm
620;
621; GFX8-LABEL: idot4_multiuse_mul1:
622; GFX8:       ; %bb.0: ; %entry
623; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
624; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
625; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
626; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
627; GFX8-NEXT:    v_mov_b32_e32 v1, s5
628; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
629; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
630; GFX8-NEXT:    flat_load_dword v3, v[0:1]
631; GFX8-NEXT:    v_mov_b32_e32 v1, s7
632; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
633; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
634; GFX8-NEXT:    flat_load_dword v0, v[0:1]
635; GFX8-NEXT:    s_load_dword s2, s[0:1], 0x0
636; GFX8-NEXT:    s_waitcnt vmcnt(1)
637; GFX8-NEXT:    v_bfe_i32 v1, v3, 0, 8
638; GFX8-NEXT:    v_bfe_i32 v4, v3, 8, 8
639; GFX8-NEXT:    v_bfe_i32 v6, v3, 16, 8
640; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 24, v3
641; GFX8-NEXT:    s_waitcnt vmcnt(0)
642; GFX8-NEXT:    v_bfe_i32 v2, v0, 0, 8
643; GFX8-NEXT:    v_bfe_i32 v5, v0, 8, 8
644; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
645; GFX8-NEXT:    v_mad_i32_i24 v8, v1, v2, s2
646; GFX8-NEXT:    v_mad_i32_i24 v4, v4, v5, v8
647; GFX8-NEXT:    v_bfe_i32 v7, v0, 16, 8
648; GFX8-NEXT:    v_mad_i32_i24 v1, v1, v2, v4
649; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
650; GFX8-NEXT:    v_mad_i32_i24 v1, v6, v7, v1
651; GFX8-NEXT:    v_mad_i32_i24 v2, v3, v0, v1
652; GFX8-NEXT:    v_mov_b32_e32 v0, s0
653; GFX8-NEXT:    v_mov_b32_e32 v1, s1
654; GFX8-NEXT:    flat_store_dword v[0:1], v2
655; GFX8-NEXT:    s_endpgm
656;
657; GFX9-NODL-LABEL: idot4_multiuse_mul1:
658; GFX9-NODL:       ; %bb.0: ; %entry
659; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
660; GFX9-NODL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
661; GFX9-NODL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
662; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
663; GFX9-NODL-NEXT:    global_load_dword v1, v0, s[4:5]
664; GFX9-NODL-NEXT:    global_load_dword v2, v0, s[6:7]
665; GFX9-NODL-NEXT:    s_load_dword s0, s[2:3], 0x0
666; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
667; GFX9-NODL-NEXT:    s_waitcnt vmcnt(1)
668; GFX9-NODL-NEXT:    v_bfe_i32 v3, v1, 0, 8
669; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
670; GFX9-NODL-NEXT:    v_bfe_i32 v4, v2, 0, 8
671; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
672; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
673; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
674; GFX9-NODL-NEXT:    v_mul_i32_i24_e32 v2, v3, v4
675; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
676; GFX9-NODL-NEXT:    v_mad_i32_i24 v3, v3, v4, s0
677; GFX9-NODL-NEXT:    v_add3_u32 v2, v5, v3, v2
678; GFX9-NODL-NEXT:    v_add3_u32 v1, v2, v6, v1
679; GFX9-NODL-NEXT:    global_store_dword v0, v1, s[2:3]
680; GFX9-NODL-NEXT:    s_endpgm
681;
682; GFX9-DL-LABEL: idot4_multiuse_mul1:
683; GFX9-DL:       ; %bb.0: ; %entry
684; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
685; GFX9-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
686; GFX9-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
687; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
688; GFX9-DL-NEXT:    global_load_dword v1, v0, s[4:5]
689; GFX9-DL-NEXT:    global_load_dword v2, v0, s[6:7]
690; GFX9-DL-NEXT:    s_load_dword s0, s[2:3], 0x0
691; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
692; GFX9-DL-NEXT:    s_waitcnt vmcnt(1)
693; GFX9-DL-NEXT:    v_bfe_i32 v3, v1, 0, 8
694; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
695; GFX9-DL-NEXT:    v_bfe_i32 v4, v2, 0, 8
696; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
697; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
698; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
699; GFX9-DL-NEXT:    v_mul_i32_i24_e32 v2, v3, v4
700; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
701; GFX9-DL-NEXT:    v_mad_i32_i24 v3, v3, v4, s0
702; GFX9-DL-NEXT:    v_add3_u32 v2, v5, v3, v2
703; GFX9-DL-NEXT:    v_add3_u32 v1, v2, v6, v1
704; GFX9-DL-NEXT:    global_store_dword v0, v1, s[2:3]
705; GFX9-DL-NEXT:    s_endpgm
706;
707; GFX10-DL-LABEL: idot4_multiuse_mul1:
708; GFX10-DL:       ; %bb.0: ; %entry
709; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
710; GFX10-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
711; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
712; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
713; GFX10-DL-NEXT:    s_clause 0x1
714; GFX10-DL-NEXT:    global_load_dword v1, v0, s[4:5]
715; GFX10-DL-NEXT:    global_load_dword v2, v0, s[6:7]
716; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
717; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
718; GFX10-DL-NEXT:    v_bfe_i32 v0, v1, 0, 8
719; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
720; GFX10-DL-NEXT:    v_bfe_i32 v3, v2, 0, 8
721; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v4, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
722; GFX10-DL-NEXT:    v_mul_i32_i24_e32 v5, v0, v3
723; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
724; GFX10-DL-NEXT:    v_mad_i32_i24 v0, v0, v3, s2
725; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
726; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
727; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0
728; GFX10-DL-NEXT:    v_add3_u32 v0, v4, v0, v5
729; GFX10-DL-NEXT:    v_add3_u32 v0, v0, v3, v1
730; GFX10-DL-NEXT:    global_store_dword v2, v0, s[0:1]
731; GFX10-DL-NEXT:    s_endpgm
732                                               <4 x i8> addrspace(1)* %src2,
733                                               i32 addrspace(1)* nocapture %dst) {
734entry:
735  %idx = call i32 @llvm.amdgcn.workitem.id.x()
736  %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
737  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
738  %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
739  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
740
741  %v1e0 = extractelement <4 x i8> %vec1, i64 0
742  %cv1e0 = sext i8 %v1e0 to i32
743  %v2e0 = extractelement <4 x i8> %vec2, i64 0
744  %cv2e0 = sext i8 %v2e0 to i32
745  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
746
747  %v1e1 = extractelement <4 x i8> %vec1, i64 1
748  %cv1e1 = sext i8 %v1e1 to i32
749  %v2e1 = extractelement <4 x i8> %vec2, i64 1
750  %cv2e1 = sext i8 %v2e1 to i32
751  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
752
753  %v1e2 = extractelement <4 x i8> %vec1, i64 2
754  %cv1e2 = sext i8 %v1e2 to i32
755  %v2e2 = extractelement <4 x i8> %vec2, i64 2
756  %cv2e2 = sext i8 %v2e2 to i32
757  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
758
759  %v1e3 = extractelement <4 x i8> %vec1, i64 3
760  %cv1e3 = sext i8 %v1e3 to i32
761  %v2e3 = extractelement <4 x i8> %vec2, i64 3
762  %cv2e3 = sext i8 %v2e3 to i32
763  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
764
765  %acc = load i32, i32 addrspace(1)* %dst, align 4
766  %add = add i32 %mul1, %acc
767  %add1 = add i32 %mul2, %add
768  %add2 = add i32 %add1, %mul1
769  %add3 = add i32 %add2, %mul3
770  %add4 = add i32 %add3, %mul4
771
772  store i32 %add4, i32 addrspace(1)* %dst, align 4
773  ret void
774}
775
776; TODO: Support this pattern.
777define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
778; GFX7-LABEL: idot4_acc32_vecMul:
779; GFX7:       ; %bb.0: ; %entry
780; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
781; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
782; GFX7-NEXT:    s_mov_b32 s3, 0xf000
783; GFX7-NEXT:    s_mov_b32 s10, 0
784; GFX7-NEXT:    s_mov_b32 s11, s3
785; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
786; GFX7-NEXT:    s_mov_b64 s[8:9], s[4:5]
787; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
788; GFX7-NEXT:    v_mov_b32_e32 v1, 0
789; GFX7-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
790; GFX7-NEXT:    s_mov_b64 s[8:9], s[6:7]
791; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
792; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
793; GFX7-NEXT:    s_mov_b32 s2, -1
794; GFX7-NEXT:    s_waitcnt vmcnt(1)
795; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 24, v2
796; GFX7-NEXT:    v_bfe_i32 v3, v2, 16, 8
797; GFX7-NEXT:    v_bfe_i32 v4, v2, 8, 8
798; GFX7-NEXT:    v_bfe_i32 v2, v2, 0, 8
799; GFX7-NEXT:    s_waitcnt vmcnt(0)
800; GFX7-NEXT:    v_ashrrev_i32_e32 v5, 24, v0
801; GFX7-NEXT:    v_bfe_i32 v6, v0, 16, 8
802; GFX7-NEXT:    v_bfe_i32 v7, v0, 8, 8
803; GFX7-NEXT:    v_bfe_i32 v0, v0, 0, 8
804; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
805; GFX7-NEXT:    v_mad_i32_i24 v0, v2, v0, s4
806; GFX7-NEXT:    v_mad_i32_i24 v0, v4, v7, v0
807; GFX7-NEXT:    v_mad_i32_i24 v0, v3, v6, v0
808; GFX7-NEXT:    v_mad_i32_i24 v0, v1, v5, v0
809; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
810; GFX7-NEXT:    s_endpgm
811;
812; GFX8-LABEL: idot4_acc32_vecMul:
813; GFX8:       ; %bb.0: ; %entry
814; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
815; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
816; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
817; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
818; GFX8-NEXT:    v_mov_b32_e32 v1, s5
819; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
820; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
821; GFX8-NEXT:    flat_load_dword v3, v[0:1]
822; GFX8-NEXT:    v_mov_b32_e32 v1, s7
823; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
824; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
825; GFX8-NEXT:    flat_load_dword v0, v[0:1]
826; GFX8-NEXT:    s_load_dword s2, s[0:1], 0x0
827; GFX8-NEXT:    s_waitcnt vmcnt(1)
828; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 8, v3
829; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 24, v3
830; GFX8-NEXT:    v_bfe_i32 v5, v3, 16, 8
831; GFX8-NEXT:    v_bfe_i32 v3, v3, 0, 8
832; GFX8-NEXT:    s_waitcnt vmcnt(0)
833; GFX8-NEXT:    v_lshrrev_b16_e32 v2, 8, v0
834; GFX8-NEXT:    v_ashrrev_i32_e32 v6, 24, v0
835; GFX8-NEXT:    v_bfe_i32 v7, v0, 16, 8
836; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 8
837; GFX8-NEXT:    v_bfe_i32 v1, v1, 0, 8
838; GFX8-NEXT:    v_bfe_i32 v2, v2, 0, 8
839; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
840; GFX8-NEXT:    v_mad_i32_i24 v0, v3, v0, s2
841; GFX8-NEXT:    v_mad_i32_i24 v0, v1, v2, v0
842; GFX8-NEXT:    v_mad_i32_i24 v0, v5, v7, v0
843; GFX8-NEXT:    v_mad_i32_i24 v2, v4, v6, v0
844; GFX8-NEXT:    v_mov_b32_e32 v0, s0
845; GFX8-NEXT:    v_mov_b32_e32 v1, s1
846; GFX8-NEXT:    flat_store_dword v[0:1], v2
847; GFX8-NEXT:    s_endpgm
848;
849; GFX9-NODL-LABEL: idot4_acc32_vecMul:
850; GFX9-NODL:       ; %bb.0: ; %entry
851; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
852; GFX9-NODL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
853; GFX9-NODL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
854; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
855; GFX9-NODL-NEXT:    global_load_dword v1, v0, s[4:5]
856; GFX9-NODL-NEXT:    global_load_dword v2, v0, s[6:7]
857; GFX9-NODL-NEXT:    s_load_dword s0, s[2:3], 0x0
858; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
859; GFX9-NODL-NEXT:    s_waitcnt vmcnt(1)
860; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v3, 8, v1
861; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
862; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v4, 8, v2
863; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
864; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
865; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
866; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v2, sext(v3), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
867; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
868; GFX9-NODL-NEXT:    v_add3_u32 v2, v5, s0, v2
869; GFX9-NODL-NEXT:    v_add3_u32 v1, v2, v6, v1
870; GFX9-NODL-NEXT:    global_store_dword v0, v1, s[2:3]
871; GFX9-NODL-NEXT:    s_endpgm
872;
873; GFX9-DL-LABEL: idot4_acc32_vecMul:
874; GFX9-DL:       ; %bb.0: ; %entry
875; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
876; GFX9-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
877; GFX9-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
878; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
879; GFX9-DL-NEXT:    global_load_dword v1, v0, s[4:5]
880; GFX9-DL-NEXT:    global_load_dword v2, v0, s[6:7]
881; GFX9-DL-NEXT:    s_load_dword s0, s[2:3], 0x0
882; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
883; GFX9-DL-NEXT:    s_waitcnt vmcnt(1)
884; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v3, 8, v1
885; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
886; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v4, 8, v2
887; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
888; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
889; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
890; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v2, sext(v3), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
891; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
892; GFX9-DL-NEXT:    v_add3_u32 v2, v5, s0, v2
893; GFX9-DL-NEXT:    v_add3_u32 v1, v2, v6, v1
894; GFX9-DL-NEXT:    global_store_dword v0, v1, s[2:3]
895; GFX9-DL-NEXT:    s_endpgm
896;
897; GFX10-DL-LABEL: idot4_acc32_vecMul:
898; GFX10-DL:       ; %bb.0: ; %entry
899; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
900; GFX10-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
901; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
902; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
903; GFX10-DL-NEXT:    s_clause 0x1
904; GFX10-DL-NEXT:    global_load_dword v1, v0, s[4:5]
905; GFX10-DL-NEXT:    global_load_dword v2, v0, s[6:7]
906; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
907; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
908; GFX10-DL-NEXT:    v_lshrrev_b16 v0, 8, v1
909; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
910; GFX10-DL-NEXT:    v_lshrrev_b16 v3, 8, v2
911; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v4, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
912; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
913; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
914; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
915; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0
916; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
917; GFX10-DL-NEXT:    v_add3_u32 v0, v4, s2, v0
918; GFX10-DL-NEXT:    v_add3_u32 v0, v0, v3, v1
919; GFX10-DL-NEXT:    global_store_dword v2, v0, s[0:1]
920; GFX10-DL-NEXT:    s_endpgm
921                                              <4 x i8> addrspace(1)* %src2,
922                                              i32 addrspace(1)* nocapture %dst) {
923entry:
924  %idx = call i32 @llvm.amdgcn.workitem.id.x()
925  %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
926  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
927  %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
928  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
929
930  %cvec1 = sext <4 x i8> %vec1 to <4 x i32>
931  %cvec2 = sext <4 x i8> %vec2 to <4 x i32>
932
933  %mul = mul <4 x i32> %cvec1, %cvec2
934  %mul0 = extractelement <4 x i32> %mul, i64 0
935  %mul1 = extractelement <4 x i32> %mul, i64 1
936  %mul2 = extractelement <4 x i32> %mul, i64 2
937  %mul3 = extractelement <4 x i32> %mul, i64 3
938
939  %acc = load i32, i32 addrspace(1)* %dst, align 4
940  %add1 = add i32 %mul0, %acc
941  %add2 = add i32 %add1, %mul1
942  %add3 = add i32 %add2, %mul2
943  %add4 = add i32 %add3, %mul3
944
945  store i32 %add4, i32 addrspace(1)* %dst, align 4
946  ret void
947}
948
949define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
950; GFX7-LABEL: idot4_acc16_vecMul:
951; GFX7:       ; %bb.0: ; %entry
952; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
953; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
954; GFX7-NEXT:    s_mov_b32 s3, 0xf000
955; GFX7-NEXT:    s_mov_b32 s10, 0
956; GFX7-NEXT:    s_mov_b32 s11, s3
957; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
958; GFX7-NEXT:    s_mov_b64 s[8:9], s[4:5]
959; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
960; GFX7-NEXT:    v_mov_b32_e32 v1, 0
961; GFX7-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
962; GFX7-NEXT:    s_mov_b64 s[8:9], s[6:7]
963; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
964; GFX7-NEXT:    s_mov_b32 s2, -1
965; GFX7-NEXT:    buffer_load_ushort v8, off, s[0:3], 0
966; GFX7-NEXT:    s_mov_b32 s4, 0xffff
967; GFX7-NEXT:    s_waitcnt vmcnt(2)
968; GFX7-NEXT:    v_bfe_i32 v1, v2, 8, 8
969; GFX7-NEXT:    v_bfe_i32 v3, v2, 0, 8
970; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
971; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
972; GFX7-NEXT:    s_waitcnt vmcnt(1)
973; GFX7-NEXT:    v_bfe_i32 v5, v0, 8, 8
974; GFX7-NEXT:    v_bfe_i32 v6, v0, 0, 8
975; GFX7-NEXT:    v_or_b32_e32 v1, v3, v1
976; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
977; GFX7-NEXT:    v_and_b32_e32 v5, s4, v6
978; GFX7-NEXT:    v_bfe_i32 v7, v0, 16, 8
979; GFX7-NEXT:    v_or_b32_e32 v3, v5, v3
980; GFX7-NEXT:    v_and_b32_e32 v6, s4, v7
981; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
982; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
983; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
984; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
985; GFX7-NEXT:    v_bfe_i32 v4, v2, 16, 8
986; GFX7-NEXT:    s_waitcnt vmcnt(0)
987; GFX7-NEXT:    v_mad_u32_u24 v1, v1, v3, v8
988; GFX7-NEXT:    v_ashrrev_i32_e32 v2, 24, v2
989; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
990; GFX7-NEXT:    v_and_b32_e32 v4, s4, v4
991; GFX7-NEXT:    v_mad_u32_u24 v1, v5, v7, v1
992; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
993; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
994; GFX7-NEXT:    v_mad_u32_u24 v1, v4, v6, v1
995; GFX7-NEXT:    v_mad_u32_u24 v0, v2, v0, v1
996; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
997; GFX7-NEXT:    s_endpgm
998;
999; GFX8-LABEL: idot4_acc16_vecMul:
1000; GFX8:       ; %bb.0: ; %entry
1001; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1002; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1003; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
1004; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1005; GFX8-NEXT:    v_mov_b32_e32 v1, s5
1006; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
1007; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1008; GFX8-NEXT:    flat_load_dword v4, v[0:1]
1009; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
1010; GFX8-NEXT:    v_mov_b32_e32 v1, s7
1011; GFX8-NEXT:    v_mov_b32_e32 v3, s1
1012; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1013; GFX8-NEXT:    v_mov_b32_e32 v2, s0
1014; GFX8-NEXT:    flat_load_dword v0, v[0:1]
1015; GFX8-NEXT:    flat_load_ushort v1, v[2:3]
1016; GFX8-NEXT:    s_waitcnt vmcnt(2)
1017; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
1018; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 8, v4
1019; GFX8-NEXT:    v_bfe_i32 v4, v4, 0, 8
1020; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 8, v5
1021; GFX8-NEXT:    v_bfe_i32 v5, v5, 0, 8
1022; GFX8-NEXT:    s_waitcnt vmcnt(1)
1023; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
1024; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 8, v0
1025; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 8
1026; GFX8-NEXT:    s_waitcnt vmcnt(0)
1027; GFX8-NEXT:    v_mad_u16 v0, v4, v0, v1
1028; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 8, v6
1029; GFX8-NEXT:    v_bfe_i32 v6, v6, 0, 8
1030; GFX8-NEXT:    v_mad_u16 v0, v7, v8, v0
1031; GFX8-NEXT:    v_mad_u16 v0, v5, v6, v0
1032; GFX8-NEXT:    v_mad_u16 v0, v9, v10, v0
1033; GFX8-NEXT:    flat_store_short v[2:3], v0
1034; GFX8-NEXT:    s_endpgm
1035;
1036; GFX9-NODL-LABEL: idot4_acc16_vecMul:
1037; GFX9-NODL:       ; %bb.0: ; %entry
1038; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1039; GFX9-NODL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
1040; GFX9-NODL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1041; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, 0xffff
1042; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1043; GFX9-NODL-NEXT:    global_load_dword v1, v0, s[4:5]
1044; GFX9-NODL-NEXT:    global_load_dword v2, v0, s[6:7]
1045; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
1046; GFX9-NODL-NEXT:    global_load_ushort v3, v0, s[2:3]
1047; GFX9-NODL-NEXT:    s_waitcnt vmcnt(2)
1048; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
1049; GFX9-NODL-NEXT:    s_waitcnt vmcnt(1)
1050; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
1051; GFX9-NODL-NEXT:    v_ashrrev_i16_e32 v7, 8, v1
1052; GFX9-NODL-NEXT:    v_and_b32_sdwa v1, v4, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1053; GFX9-NODL-NEXT:    v_ashrrev_i16_e32 v8, 8, v2
1054; GFX9-NODL-NEXT:    v_and_b32_sdwa v2, v4, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1055; GFX9-NODL-NEXT:    v_lshl_or_b32 v2, v8, 16, v2
1056; GFX9-NODL-NEXT:    v_lshl_or_b32 v1, v7, 16, v1
1057; GFX9-NODL-NEXT:    v_ashrrev_i16_e32 v10, 8, v6
1058; GFX9-NODL-NEXT:    v_and_b32_sdwa v6, v4, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1059; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v1, v1, v2
1060; GFX9-NODL-NEXT:    v_ashrrev_i16_e32 v9, 8, v5
1061; GFX9-NODL-NEXT:    v_and_b32_sdwa v4, v4, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1062; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
1063; GFX9-NODL-NEXT:    v_add_u16_e32 v3, v1, v3
1064; GFX9-NODL-NEXT:    v_lshl_or_b32 v5, v10, 16, v6
1065; GFX9-NODL-NEXT:    v_lshl_or_b32 v4, v9, 16, v4
1066; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v2, v4, v5
1067; GFX9-NODL-NEXT:    v_add_u16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1068; GFX9-NODL-NEXT:    v_add_u16_e32 v1, v1, v2
1069; GFX9-NODL-NEXT:    v_add_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1070; GFX9-NODL-NEXT:    global_store_short v0, v1, s[2:3]
1071; GFX9-NODL-NEXT:    s_endpgm
1072;
1073; GFX9-DL-LABEL: idot4_acc16_vecMul:
1074; GFX9-DL:       ; %bb.0: ; %entry
1075; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1076; GFX9-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
1077; GFX9-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1078; GFX9-DL-NEXT:    v_mov_b32_e32 v4, 0xffff
1079; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1080; GFX9-DL-NEXT:    global_load_dword v1, v0, s[4:5]
1081; GFX9-DL-NEXT:    global_load_dword v2, v0, s[6:7]
1082; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
1083; GFX9-DL-NEXT:    global_load_ushort v3, v0, s[2:3]
1084; GFX9-DL-NEXT:    s_waitcnt vmcnt(2)
1085; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
1086; GFX9-DL-NEXT:    s_waitcnt vmcnt(1)
1087; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
1088; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 8, v1
1089; GFX9-DL-NEXT:    v_and_b32_sdwa v1, v4, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1090; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v8, 8, v2
1091; GFX9-DL-NEXT:    v_and_b32_sdwa v2, v4, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1092; GFX9-DL-NEXT:    v_lshl_or_b32 v2, v8, 16, v2
1093; GFX9-DL-NEXT:    v_lshl_or_b32 v1, v7, 16, v1
1094; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 8, v6
1095; GFX9-DL-NEXT:    v_and_b32_sdwa v6, v4, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1096; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v1, v1, v2
1097; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 8, v5
1098; GFX9-DL-NEXT:    v_and_b32_sdwa v4, v4, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1099; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
1100; GFX9-DL-NEXT:    v_add_u16_e32 v3, v1, v3
1101; GFX9-DL-NEXT:    v_lshl_or_b32 v5, v10, 16, v6
1102; GFX9-DL-NEXT:    v_lshl_or_b32 v4, v9, 16, v4
1103; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v2, v4, v5
1104; GFX9-DL-NEXT:    v_add_u16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1105; GFX9-DL-NEXT:    v_add_u16_e32 v1, v1, v2
1106; GFX9-DL-NEXT:    v_add_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
1107; GFX9-DL-NEXT:    global_store_short v0, v1, s[2:3]
1108; GFX9-DL-NEXT:    s_endpgm
1109;
1110; GFX10-DL-LABEL: idot4_acc16_vecMul:
1111; GFX10-DL:       ; %bb.0: ; %entry
1112; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1113; GFX10-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1114; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1115; GFX10-DL-NEXT:    v_mov_b32_e32 v4, 0xffff
1116; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1117; GFX10-DL-NEXT:    s_clause 0x1
1118; GFX10-DL-NEXT:    global_load_dword v1, v0, s[4:5]
1119; GFX10-DL-NEXT:    global_load_dword v2, v0, s[6:7]
1120; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
1121; GFX10-DL-NEXT:    global_load_ushort v3, v0, s[0:1]
1122; GFX10-DL-NEXT:    s_waitcnt vmcnt(2)
1123; GFX10-DL-NEXT:    v_ashrrev_i16 v5, 8, v1
1124; GFX10-DL-NEXT:    v_and_b32_sdwa v8, v4, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1125; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
1126; GFX10-DL-NEXT:    v_ashrrev_i16 v6, 8, v2
1127; GFX10-DL-NEXT:    v_and_b32_sdwa v7, v4, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1128; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
1129; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
1130; GFX10-DL-NEXT:    v_lshl_or_b32 v5, v5, 16, v8
1131; GFX10-DL-NEXT:    v_lshl_or_b32 v6, v6, 16, v7
1132; GFX10-DL-NEXT:    v_ashrrev_i16 v7, 8, v1
1133; GFX10-DL-NEXT:    v_and_b32_sdwa v1, v4, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1134; GFX10-DL-NEXT:    v_ashrrev_i16 v8, 8, v2
1135; GFX10-DL-NEXT:    v_and_b32_sdwa v2, v4, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
1136; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v4, v5, v6
1137; GFX10-DL-NEXT:    v_lshl_or_b32 v1, v7, 16, v1
1138; GFX10-DL-NEXT:    v_lshl_or_b32 v2, v8, 16, v2
1139; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
1140; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
1141; GFX10-DL-NEXT:    v_add_nc_u16 v3, v4, v3
1142; GFX10-DL-NEXT:    v_pk_mul_lo_u16 v1, v1, v2
1143; GFX10-DL-NEXT:    v_add_nc_u16 v2, v3, v5
1144; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
1145; GFX10-DL-NEXT:    v_add_nc_u16 v1, v2, v1
1146; GFX10-DL-NEXT:    v_add_nc_u16 v1, v1, v3
1147; GFX10-DL-NEXT:    global_store_short v0, v1, s[0:1]
1148; GFX10-DL-NEXT:    s_endpgm
1149                                              <4 x i8> addrspace(1)* %src2,
1150                                              i16 addrspace(1)* nocapture %dst) {
1151entry:
1152  %idx = call i32 @llvm.amdgcn.workitem.id.x()
1153  %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
1154  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
1155  %gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
1156  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
1157
1158  %cvec1 = sext <4 x i8> %vec1 to <4 x i16>
1159  %cvec2 = sext <4 x i8> %vec2 to <4 x i16>
1160
1161  %mul = mul <4 x i16> %cvec1, %cvec2
1162  %mul0 = extractelement <4 x i16> %mul, i64 0
1163  %mul1 = extractelement <4 x i16> %mul, i64 1
1164  %mul2 = extractelement <4 x i16> %mul, i64 2
1165  %mul3 = extractelement <4 x i16> %mul, i64 3
1166
1167  %acc = load i16, i16 addrspace(1)* %dst, align 4
1168  %add1 = add i16 %mul0, %acc
1169  %add2 = add i16 %add1, %mul1
1170  %add3 = add i16 %add2, %mul2
1171  %add4 = add i16 %add3, %mul3
1172
1173  store i16 %add4, i16 addrspace(1)* %dst, align 4
1174  ret void
1175}
1176
1177declare i32 @llvm.amdgcn.workitem.id.x()
1178