1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX8 %s
4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-NODL %s
5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-DL %s
6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-DL %s
7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-DL %s
8
9; add(mul(S0.x, S1.y),
10;     add (mul (S0.y, S1.y), S3)) -> v_dot2_{I|U}32_{I|U}16(S1, S2, S3)
11
12define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1,
13; GFX7-LABEL: udot2:
14; GFX7:       ; %bb.0: ; %entry
15; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
16; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
17; GFX7-NEXT:    s_mov_b32 s8, 0xffff
18; GFX7-NEXT:    s_mov_b32 s3, 0xf000
19; GFX7-NEXT:    s_mov_b32 s2, -1
20; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
22; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
23; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
24; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
25; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
26; GFX7-NEXT:    s_and_b32 s4, s4, s8
27; GFX7-NEXT:    s_and_b32 s5, s5, s8
28; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
29; GFX7-NEXT:    v_mov_b32_e32 v0, s6
30; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
31; GFX7-NEXT:    v_mov_b32_e32 v1, s8
32; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
33; GFX7-NEXT:    v_mov_b32_e32 v1, s4
34; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
35; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
36; GFX7-NEXT:    s_endpgm
37;
38; GFX8-LABEL: udot2:
39; GFX8:       ; %bb.0: ; %entry
40; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
41; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
42; GFX8-NEXT:    s_mov_b32 s2, 0xffff
43; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
45; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
46; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
47; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
48; GFX8-NEXT:    s_and_b32 s6, s3, s2
49; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
50; GFX8-NEXT:    s_and_b32 s2, s4, s2
51; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
52; GFX8-NEXT:    v_mov_b32_e32 v0, s5
53; GFX8-NEXT:    v_mov_b32_e32 v1, s3
54; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
55; GFX8-NEXT:    v_mov_b32_e32 v1, s6
56; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
57; GFX8-NEXT:    v_mov_b32_e32 v0, s0
58; GFX8-NEXT:    v_mov_b32_e32 v1, s1
59; GFX8-NEXT:    flat_store_dword v[0:1], v2
60; GFX8-NEXT:    s_endpgm
61;
62; GFX9-NODL-LABEL: udot2:
63; GFX9-NODL:       ; %bb.0: ; %entry
64; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
65; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
66; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
67; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
68; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
69; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
70; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
71; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
72; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
73; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
74; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
75; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
76; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
77; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
78; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
79; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
80; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
81; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
82; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
83; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
84; GFX9-NODL-NEXT:    s_endpgm
85;
86; GFX9-DL-LABEL: udot2:
87; GFX9-DL:       ; %bb.0: ; %entry
88; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
89; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
90; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
91; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
92; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
93; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
94; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
95; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
96; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
97; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s4, v0, v1
98; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
99; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
100; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
101; GFX9-DL-NEXT:    s_endpgm
102;
103; GFX10-DL-LABEL: udot2:
104; GFX10-DL:       ; %bb.0: ; %entry
105; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
106; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
107; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
108; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
109; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
110; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
111; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
112; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
113; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
114; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s4, s3, v0
115; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
116; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
117; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
118; GFX10-DL-NEXT:    s_endpgm
119                                 <2 x i16> addrspace(1)* %src2,
120                                 i32 addrspace(1)* nocapture %dst) {
121entry:
122  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
123  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
124
125  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
126  %conv = zext i16 %s1.elt1 to i32
127  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
128  %conv2 = zext i16 %s2.elt1 to i32
129  %mul1 = mul nuw i32 %conv2, %conv
130
131  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
132  %conv3 = zext i16 %s1.elt2 to i32
133  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
134  %conv4 = zext i16 %s2.elt2 to i32
135  %mul2 = mul nuw i32 %conv4, %conv3
136
137  %s3 = load i32, i32 addrspace(1)* %dst, align 4
138  %add = add i32 %mul2, %s3
139  %add6 = add i32 %add, %mul1
140  store i32 %add6, i32 addrspace(1)* %dst, align 4
141  ret void
142}
143
144; TODO: Support this pattern
145;      add(S3,
146;          add (mul (S0.y, S1.y), mul (S0.y, S1.y))) -> v_dot2_{I|U}32_{I|U}16(S1, S2, S3)
147define amdgpu_kernel void @udot2_MulMul(<2 x i16> addrspace(1)* %src1,
148; GFX7-LABEL: udot2_MulMul:
149; GFX7:       ; %bb.0: ; %entry
150; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
151; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
152; GFX7-NEXT:    s_mov_b32 s8, 0xffff
153; GFX7-NEXT:    s_mov_b32 s3, 0xf000
154; GFX7-NEXT:    s_mov_b32 s2, -1
155; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
156; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
157; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
158; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
159; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
160; GFX7-NEXT:    s_and_b32 s4, s4, s8
161; GFX7-NEXT:    v_mov_b32_e32 v0, s4
162; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
163; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
164; GFX7-NEXT:    s_and_b32 s5, s5, s8
165; GFX7-NEXT:    v_mul_u32_u24_e32 v0, s5, v0
166; GFX7-NEXT:    v_mov_b32_e32 v1, s6
167; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
168; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
169; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
170; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
171; GFX7-NEXT:    s_endpgm
172;
173; GFX8-LABEL: udot2_MulMul:
174; GFX8:       ; %bb.0: ; %entry
175; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
176; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
177; GFX8-NEXT:    s_mov_b32 s2, 0xffff
178; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
179; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
180; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
181; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
182; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
183; GFX8-NEXT:    s_and_b32 s6, s3, s2
184; GFX8-NEXT:    s_and_b32 s2, s4, s2
185; GFX8-NEXT:    v_mov_b32_e32 v0, s6
186; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
187; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
188; GFX8-NEXT:    v_mov_b32_e32 v1, s3
189; GFX8-NEXT:    v_mul_u32_u24_e32 v0, s2, v0
190; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
191; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s5, v0
192; GFX8-NEXT:    v_mov_b32_e32 v0, s0
193; GFX8-NEXT:    v_mov_b32_e32 v1, s1
194; GFX8-NEXT:    flat_store_dword v[0:1], v2
195; GFX8-NEXT:    s_endpgm
196;
197; GFX9-NODL-LABEL: udot2_MulMul:
198; GFX9-NODL:       ; %bb.0: ; %entry
199; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
200; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
201; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
202; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
203; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
204; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
205; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
206; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
207; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
208; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
209; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s6
210; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
211; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
212; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
213; GFX9-NODL-NEXT:    v_mul_u32_u24_e32 v0, s2, v0
214; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
215; GFX9-NODL-NEXT:    v_add_u32_e32 v2, s5, v0
216; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
217; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
218; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
219; GFX9-NODL-NEXT:    s_endpgm
220;
221; GFX9-DL-LABEL: udot2_MulMul:
222; GFX9-DL:       ; %bb.0: ; %entry
223; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
224; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
225; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
226; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
227; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
228; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
229; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
230; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
231; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
232; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
233; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
234; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
235; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
236; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
237; GFX9-DL-NEXT:    v_mul_u32_u24_e32 v0, s2, v0
238; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
239; GFX9-DL-NEXT:    v_add_u32_e32 v2, s5, v0
240; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
241; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
242; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
243; GFX9-DL-NEXT:    s_endpgm
244;
245; GFX10-DL-LABEL: udot2_MulMul:
246; GFX10-DL:       ; %bb.0: ; %entry
247; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
248; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
249; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
250; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
251; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
252; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
253; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
254; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
255; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
256; GFX10-DL-NEXT:    s_and_b32 s6, s3, s2
257; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
258; GFX10-DL-NEXT:    s_lshr_b32 s3, s3, 16
259; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 16
260; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v0, s2, s6
261; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s4, s3, v0
262; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, s5, v0
263; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
264; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
265; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
266; GFX10-DL-NEXT:    s_endpgm
267                                        <2 x i16> addrspace(1)* %src2,
268                                        i32 addrspace(1)* nocapture %dst) {
269entry:
270  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
271  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
272
273  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
274  %conv = zext i16 %s1.elt1 to i32
275  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
276  %conv2 = zext i16 %s2.elt1 to i32
277  %mul1 = mul nuw i32 %conv2, %conv
278
279  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
280  %conv3 = zext i16 %s1.elt2 to i32
281  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
282  %conv4 = zext i16 %s2.elt2 to i32
283  %mul2 = mul nuw i32 %conv4, %conv3
284  %s3 = load i32, i32 addrspace(1)* %dst, align 4
285  %add = add i32 %mul2, %mul1
286  %add6 = add i32 %add, %s3
287  store i32 %add6, i32 addrspace(1)* %dst, align 4
288  ret void
289}
290
291define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
292; GFX7-LABEL: idot2:
293; GFX7:       ; %bb.0: ; %entry
294; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
295; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
296; GFX7-NEXT:    s_mov_b32 s3, 0xf000
297; GFX7-NEXT:    s_mov_b32 s2, -1
298; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
299; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
300; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
301; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
302; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
303; GFX7-NEXT:    s_sext_i32_i16 s7, s4
304; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
305; GFX7-NEXT:    s_sext_i32_i16 s8, s5
306; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
307; GFX7-NEXT:    v_mov_b32_e32 v0, s4
308; GFX7-NEXT:    v_mov_b32_e32 v1, s6
309; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
310; GFX7-NEXT:    v_mov_b32_e32 v1, s7
311; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
312; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
313; GFX7-NEXT:    s_endpgm
314;
315; GFX8-LABEL: idot2:
316; GFX8:       ; %bb.0: ; %entry
317; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
318; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
319; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
320; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
321; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
322; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
323; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
324; GFX8-NEXT:    s_sext_i32_i16 s5, s2
325; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
326; GFX8-NEXT:    s_sext_i32_i16 s6, s3
327; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
328; GFX8-NEXT:    v_mov_b32_e32 v0, s4
329; GFX8-NEXT:    v_mov_b32_e32 v1, s2
330; GFX8-NEXT:    v_mov_b32_e32 v2, s5
331; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
332; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
333; GFX8-NEXT:    v_mov_b32_e32 v0, s0
334; GFX8-NEXT:    v_mov_b32_e32 v1, s1
335; GFX8-NEXT:    flat_store_dword v[0:1], v2
336; GFX8-NEXT:    s_endpgm
337;
338; GFX9-NODL-LABEL: idot2:
339; GFX9-NODL:       ; %bb.0: ; %entry
340; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
341; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
342; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
343; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
344; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
345; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
346; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
347; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
348; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
349; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
350; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
351; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
352; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
353; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
354; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
355; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
356; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
357; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
358; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
359; GFX9-NODL-NEXT:    s_endpgm
360;
361; GFX9-DL-LABEL: idot2:
362; GFX9-DL:       ; %bb.0: ; %entry
363; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
364; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
365; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
366; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
367; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
368; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
369; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
370; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
371; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
372; GFX9-DL-NEXT:    v_dot2_i32_i16 v2, s4, v0, v1
373; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
374; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
375; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
376; GFX9-DL-NEXT:    s_endpgm
377;
378; GFX10-DL-LABEL: idot2:
379; GFX10-DL:       ; %bb.0: ; %entry
380; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
381; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
382; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
383; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
384; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
385; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
386; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
387; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
388; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
389; GFX10-DL-NEXT:    v_dot2_i32_i16 v2, s4, s3, v0
390; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
391; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
392; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
393; GFX10-DL-NEXT:    s_endpgm
394                                 <2 x i16> addrspace(1)* %src2,
395                                 i32 addrspace(1)* nocapture %dst) {
396entry:
397  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
398  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
399
400  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
401  %conv = sext i16 %s1.elt1 to i32
402  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
403  %conv2 = sext i16 %s2.elt1 to i32
404  %mul1 = mul nuw i32 %conv2, %conv
405
406  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
407  %conv3 = sext i16 %s1.elt2 to i32
408  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
409  %conv4 = sext i16 %s2.elt2 to i32
410  %mul2 = mul nuw i32 %conv4, %conv3
411
412  %s3 = load i32, i32 addrspace(1)* %dst, align 4
413  %add = add i32 %mul2, %s3
414  %add6 = add i32 %add, %mul1
415  store i32 %add6, i32 addrspace(1)* %dst, align 4
416  ret void
417}
418
419define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
420; GFX7-LABEL: idot2_MixedTypedMul:
421; GFX7:       ; %bb.0: ; %entry
422; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
423; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
424; GFX7-NEXT:    s_mov_b32 s3, 0xf000
425; GFX7-NEXT:    s_mov_b32 s2, -1
426; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
427; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
428; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
429; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
430; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
431; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
432; GFX7-NEXT:    s_lshr_b32 s8, s5, 16
433; GFX7-NEXT:    s_sext_i32_i16 s4, s4
434; GFX7-NEXT:    v_mov_b32_e32 v0, s7
435; GFX7-NEXT:    v_mov_b32_e32 v1, s6
436; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v0, v1
437; GFX7-NEXT:    s_sext_i32_i16 s5, s5
438; GFX7-NEXT:    v_mov_b32_e32 v1, s4
439; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
440; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
441; GFX7-NEXT:    s_endpgm
442;
443; GFX8-LABEL: idot2_MixedTypedMul:
444; GFX8:       ; %bb.0: ; %entry
445; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
446; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
447; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
448; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
449; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
450; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
451; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
452; GFX8-NEXT:    s_sext_i32_i16 s5, s2
453; GFX8-NEXT:    s_lshr_b32 s2, s2, 16
454; GFX8-NEXT:    s_sext_i32_i16 s6, s3
455; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
456; GFX8-NEXT:    v_mov_b32_e32 v0, s4
457; GFX8-NEXT:    v_mov_b32_e32 v1, s2
458; GFX8-NEXT:    v_mov_b32_e32 v2, s5
459; GFX8-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
460; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
461; GFX8-NEXT:    v_mov_b32_e32 v0, s0
462; GFX8-NEXT:    v_mov_b32_e32 v1, s1
463; GFX8-NEXT:    flat_store_dword v[0:1], v2
464; GFX8-NEXT:    s_endpgm
465;
466; GFX9-NODL-LABEL: idot2_MixedTypedMul:
467; GFX9-NODL:       ; %bb.0: ; %entry
468; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
469; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
470; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
471; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
472; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
473; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
474; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
475; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
476; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 16
477; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
478; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
479; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
480; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
481; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
482; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
483; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
484; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
485; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
486; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
487; GFX9-NODL-NEXT:    s_endpgm
488;
489; GFX9-DL-LABEL: idot2_MixedTypedMul:
490; GFX9-DL:       ; %bb.0: ; %entry
491; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
492; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
493; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
494; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
495; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
496; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
497; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
498; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
499; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 16
500; GFX9-DL-NEXT:    s_sext_i32_i16 s6, s3
501; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
502; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
503; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
504; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
505; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
506; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
507; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
508; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
509; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
510; GFX9-DL-NEXT:    s_endpgm
511;
512; GFX10-DL-LABEL: idot2_MixedTypedMul:
513; GFX10-DL:       ; %bb.0: ; %entry
514; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
515; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
516; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
517; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
518; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
519; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
520; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
521; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
522; GFX10-DL-NEXT:    s_lshr_b32 s5, s2, 16
523; GFX10-DL-NEXT:    s_lshr_b32 s6, s3, 16
524; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
525; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s2
526; GFX10-DL-NEXT:    s_sext_i32_i16 s3, s3
527; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s5, v0
528; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v0
529; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
530; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
531; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
532; GFX10-DL-NEXT:    s_endpgm
533                                               <2 x i16> addrspace(1)* %src2,
534                                               i32 addrspace(1)* nocapture %dst) {
535entry:
536  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
537  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
538
539  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
540  %conv = sext i16 %s1.elt1 to i32
541  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
542  %conv2 = sext i16 %s2.elt1 to i32
543  %mul1 = mul nuw i32 %conv2, %conv
544
545  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
546  %conv3 = zext i16 %s1.elt2 to i32
547  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
548  %conv4 = zext i16 %s2.elt2 to i32
549  %mul2 = mul nuw i32 %conv4, %conv3
550
551  %s3 = load i32, i32 addrspace(1)* %dst, align 4
552  %add = add i32 %mul2, %s3
553  %add6 = add i32 %add, %mul1
554  store i32 %add6, i32 addrspace(1)* %dst, align 4
555  ret void
556}
557
558define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1,
559; GFX7-LABEL: udot2_alt_AddOperands:
560; GFX7:       ; %bb.0: ; %entry
561; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
562; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
563; GFX7-NEXT:    s_mov_b32 s8, 0xffff
564; GFX7-NEXT:    s_mov_b32 s3, 0xf000
565; GFX7-NEXT:    s_mov_b32 s2, -1
566; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
567; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
568; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
569; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
570; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
571; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
572; GFX7-NEXT:    s_and_b32 s4, s4, s8
573; GFX7-NEXT:    s_and_b32 s5, s5, s8
574; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
575; GFX7-NEXT:    v_mov_b32_e32 v0, s6
576; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
577; GFX7-NEXT:    v_mov_b32_e32 v1, s8
578; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
579; GFX7-NEXT:    v_mov_b32_e32 v1, s4
580; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
581; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
582; GFX7-NEXT:    s_endpgm
583;
584; GFX8-LABEL: udot2_alt_AddOperands:
585; GFX8:       ; %bb.0: ; %entry
586; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
587; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
588; GFX8-NEXT:    s_mov_b32 s2, 0xffff
589; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
590; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
591; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
592; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
593; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
594; GFX8-NEXT:    s_and_b32 s6, s3, s2
595; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
596; GFX8-NEXT:    s_and_b32 s2, s4, s2
597; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
598; GFX8-NEXT:    v_mov_b32_e32 v0, s5
599; GFX8-NEXT:    v_mov_b32_e32 v1, s3
600; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
601; GFX8-NEXT:    v_mov_b32_e32 v1, s6
602; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
603; GFX8-NEXT:    v_mov_b32_e32 v0, s0
604; GFX8-NEXT:    v_mov_b32_e32 v1, s1
605; GFX8-NEXT:    flat_store_dword v[0:1], v2
606; GFX8-NEXT:    s_endpgm
607;
608; GFX9-NODL-LABEL: udot2_alt_AddOperands:
609; GFX9-NODL:       ; %bb.0: ; %entry
610; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
611; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
612; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
613; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
614; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
615; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
616; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
617; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
618; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
619; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
620; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
621; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
622; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
623; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
624; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
625; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
626; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
627; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
628; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
629; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
630; GFX9-NODL-NEXT:    s_endpgm
631;
632; GFX9-DL-LABEL: udot2_alt_AddOperands:
633; GFX9-DL:       ; %bb.0: ; %entry
634; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
635; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
636; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
637; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
638; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
639; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
640; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
641; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
642; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
643; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s4, v0, v1
644; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
645; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
646; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
647; GFX9-DL-NEXT:    s_endpgm
648;
649; GFX10-DL-LABEL: udot2_alt_AddOperands:
650; GFX10-DL:       ; %bb.0: ; %entry
651; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
652; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
653; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
654; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
655; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
656; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
657; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
658; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
659; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
660; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s4, s3, v0
661; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
662; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
663; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
664; GFX10-DL-NEXT:    s_endpgm
665                                                 <2 x i16> addrspace(1)* %src2,
666                                                 i32 addrspace(1)* nocapture %dst) {
667entry:
668  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
669  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
670
671  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
672  %conv = zext i16 %s1.elt1 to i32
673  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
674  %conv2 = zext i16 %s2.elt1 to i32
675  %mul1 = mul nuw i32 %conv2, %conv
676
677  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
678  %conv3 = zext i16 %s1.elt2 to i32
679  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
680  %conv4 = zext i16 %s2.elt2 to i32
681  %mul2 = mul nuw i32 %conv4, %conv3
682
683  %s3 = load i32, i32 addrspace(1)* %dst, align 4
684  %add = add i32 %s3, %mul2
685  %add6 = add i32 %mul1, %add
686  store i32 %add6, i32 addrspace(1)* %dst, align 4
687  ret void
688}
689
690define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
691; GFX7-LABEL: idot2_MixedExt:
692; GFX7:       ; %bb.0: ; %entry
693; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
694; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
695; GFX7-NEXT:    s_mov_b32 s3, 0xf000
696; GFX7-NEXT:    s_mov_b32 s2, -1
697; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
698; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
699; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
700; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
701; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
702; GFX7-NEXT:    s_sext_i32_i16 s7, s4
703; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
704; GFX7-NEXT:    s_and_b32 s8, s5, 0xffff
705; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
706; GFX7-NEXT:    v_mov_b32_e32 v0, s4
707; GFX7-NEXT:    v_mov_b32_e32 v1, s6
708; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
709; GFX7-NEXT:    v_mov_b32_e32 v1, s7
710; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
711; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
712; GFX7-NEXT:    s_endpgm
713;
714; GFX8-LABEL: idot2_MixedExt:
715; GFX8:       ; %bb.0: ; %entry
716; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
717; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
718; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
719; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
720; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
721; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
722; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
723; GFX8-NEXT:    s_sext_i32_i16 s5, s2
724; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
725; GFX8-NEXT:    s_and_b32 s6, s3, 0xffff
726; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
727; GFX8-NEXT:    v_mov_b32_e32 v0, s4
728; GFX8-NEXT:    v_mov_b32_e32 v1, s2
729; GFX8-NEXT:    v_mov_b32_e32 v2, s5
730; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
731; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
732; GFX8-NEXT:    v_mov_b32_e32 v0, s0
733; GFX8-NEXT:    v_mov_b32_e32 v1, s1
734; GFX8-NEXT:    flat_store_dword v[0:1], v2
735; GFX8-NEXT:    s_endpgm
736;
737; GFX9-NODL-LABEL: idot2_MixedExt:
738; GFX9-NODL:       ; %bb.0: ; %entry
739; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
740; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
741; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
742; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
743; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
744; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
745; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
746; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
747; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
748; GFX9-NODL-NEXT:    s_and_b32 s6, s3, 0xffff
749; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
750; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
751; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
752; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
753; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
754; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
755; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
756; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
757; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
758; GFX9-NODL-NEXT:    s_endpgm
759;
760; GFX9-DL-LABEL: idot2_MixedExt:
761; GFX9-DL:       ; %bb.0: ; %entry
762; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
763; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
764; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
765; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
766; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
767; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
768; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
769; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
770; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
771; GFX9-DL-NEXT:    s_and_b32 s6, s3, 0xffff
772; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
773; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
774; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
775; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
776; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
777; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
778; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
779; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
780; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
781; GFX9-DL-NEXT:    s_endpgm
782;
783; GFX10-DL-LABEL: idot2_MixedExt:
784; GFX10-DL:       ; %bb.0: ; %entry
785; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
786; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
787; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
788; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
789; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
790; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
791; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
792; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
793; GFX10-DL-NEXT:    s_ashr_i32 s5, s2, 16
794; GFX10-DL-NEXT:    s_ashr_i32 s6, s3, 16
795; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
796; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s2
797; GFX10-DL-NEXT:    s_and_b32 s3, s3, 0xffff
798; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s5, v0
799; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v0
800; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
801; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
802; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
803; GFX10-DL-NEXT:    s_endpgm
804                                          <2 x i16> addrspace(1)* %src2,
805                                          i32 addrspace(1)* nocapture %dst) {
806entry:
807  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
808  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
809
810  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
811  %conv = sext i16 %s1.elt1 to i32
812  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
813  %conv2 = zext i16 %s2.elt1 to i32
814  %mul1 = mul nuw i32 %conv2, %conv
815
816  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
817  %conv3 = sext i16 %s1.elt2 to i32
818  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
819  %conv4 = sext i16 %s2.elt2 to i32
820  %mul2 = mul nuw i32 %conv4, %conv3
821
822  %s3 = load i32, i32 addrspace(1)* %dst, align 4
823  %add = add i32 %mul2, %s3
824  %add6 = add i32 %add, %mul1
825  store i32 %add6, i32 addrspace(1)* %dst, align 4
826  ret void
827}
828
829define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
830; GFX7-LABEL: notudot2_SameVec:
831; GFX7:       ; %bb.0: ; %entry
832; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
833; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
834; GFX7-NEXT:    s_mov_b32 s3, 0xf000
835; GFX7-NEXT:    s_mov_b32 s2, -1
836; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
837; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
838; GFX7-NEXT:    s_load_dword s7, s[0:1], 0x0
839; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
840; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
841; GFX7-NEXT:    s_lshr_b32 s5, s6, 16
842; GFX7-NEXT:    v_mov_b32_e32 v0, s7
843; GFX7-NEXT:    s_and_b32 s4, s4, 0xffff
844; GFX7-NEXT:    v_mad_u32_u24 v0, s5, s5, v0
845; GFX7-NEXT:    v_mad_u32_u24 v0, s4, s4, v0
846; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
847; GFX7-NEXT:    s_endpgm
848;
849; GFX8-LABEL: notudot2_SameVec:
850; GFX8:       ; %bb.0: ; %entry
851; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
852; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
853; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
854; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
855; GFX8-NEXT:    s_load_dword s3, s[0:1], 0x0
856; GFX8-NEXT:    s_load_dword s4, s[4:5], 0x0
857; GFX8-NEXT:    v_mov_b32_e32 v0, s0
858; GFX8-NEXT:    v_mov_b32_e32 v1, s1
859; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
860; GFX8-NEXT:    s_lshr_b32 s0, s2, 16
861; GFX8-NEXT:    v_mov_b32_e32 v2, s3
862; GFX8-NEXT:    s_and_b32 s1, s4, 0xffff
863; GFX8-NEXT:    v_mad_u32_u24 v2, s0, s0, v2
864; GFX8-NEXT:    v_mad_u32_u24 v2, s1, s1, v2
865; GFX8-NEXT:    flat_store_dword v[0:1], v2
866; GFX8-NEXT:    s_endpgm
867;
868; GFX9-NODL-LABEL: notudot2_SameVec:
869; GFX9-NODL:       ; %bb.0: ; %entry
870; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
871; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
872; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
873; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
874; GFX9-NODL-NEXT:    s_load_dword s3, s[0:1], 0x0
875; GFX9-NODL-NEXT:    s_load_dword s4, s[4:5], 0x0
876; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
877; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
878; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
879; GFX9-NODL-NEXT:    s_lshr_b32 s0, s2, 16
880; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s3
881; GFX9-NODL-NEXT:    s_and_b32 s1, s4, 0xffff
882; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, s0, v2
883; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s1, s1, v2
884; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
885; GFX9-NODL-NEXT:    s_endpgm
886;
887; GFX9-DL-LABEL: notudot2_SameVec:
888; GFX9-DL:       ; %bb.0: ; %entry
889; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
890; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
891; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
892; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
893; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
894; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
895; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
896; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
897; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
898; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 16
899; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
900; GFX9-DL-NEXT:    s_and_b32 s1, s4, 0xffff
901; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, s0, v2
902; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, s1, v2
903; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
904; GFX9-DL-NEXT:    s_endpgm
905;
906; GFX10-DL-LABEL: notudot2_SameVec:
907; GFX10-DL:       ; %bb.0: ; %entry
908; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
909; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
910; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
911; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
912; GFX10-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
913; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
914; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
915; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
916; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 16
917; GFX10-DL-NEXT:    s_and_b32 s3, s3, 0xffff
918; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s2, s2, s4
919; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s3, v0
920; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
921; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
922; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
923; GFX10-DL-NEXT:    s_endpgm
924                                            <2 x i16> addrspace(1)* %src2,
925                                            i32 addrspace(1)* nocapture %dst) {
926entry:
927  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
928  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
929
930  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
931  %conv = zext i16 %s1.elt1 to i32
932  %s2.elt1 = extractelement <2 x i16> %vec1, i64 0
933  %conv2 = zext i16 %s2.elt1 to i32
934  %mul1 = mul i32 %conv2, %conv
935
936  %s1.elt2 = extractelement <2 x i16> %vec2, i64 1
937  %conv3 = zext i16 %s1.elt2 to i32
938  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
939  %conv4 = zext i16 %s2.elt2 to i32
940  %mul2 = mul i32 %conv4, %conv3
941
942  %s3 = load i32, i32 addrspace(1)* %dst, align 4
943  %add = add i32 %mul2, %s3
944  %add6 = add i32 %add, %mul1
945  store i32 %add6, i32 addrspace(1)* %dst, align 4
946  ret void
947}
948
949define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1,
950; GFX7-LABEL: udot2_v4i16:
951; GFX7:       ; %bb.0: ; %entry
952; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
953; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
954; GFX7-NEXT:    s_mov_b32 s8, 0xffff
955; GFX7-NEXT:    s_mov_b32 s3, 0xf000
956; GFX7-NEXT:    s_mov_b32 s2, -1
957; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
958; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
959; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
960; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
961; GFX7-NEXT:    s_and_b32 s6, s4, s8
962; GFX7-NEXT:    s_and_b32 s7, s5, s8
963; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
964; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
965; GFX7-NEXT:    s_lshr_b32 s5, s5, 16
966; GFX7-NEXT:    v_mov_b32_e32 v0, s4
967; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
968; GFX7-NEXT:    v_mov_b32_e32 v1, s8
969; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
970; GFX7-NEXT:    v_mov_b32_e32 v1, s6
971; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
972; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
973; GFX7-NEXT:    s_endpgm
974;
975; GFX8-LABEL: udot2_v4i16:
976; GFX8:       ; %bb.0: ; %entry
977; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
978; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
979; GFX8-NEXT:    s_mov_b32 s2, 0xffff
980; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
981; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
982; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
983; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
984; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
985; GFX8-NEXT:    s_and_b32 s6, s3, s2
986; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
987; GFX8-NEXT:    s_and_b32 s2, s4, s2
988; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
989; GFX8-NEXT:    v_mov_b32_e32 v0, s5
990; GFX8-NEXT:    v_mov_b32_e32 v1, s3
991; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
992; GFX8-NEXT:    v_mov_b32_e32 v1, s6
993; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
994; GFX8-NEXT:    v_mov_b32_e32 v0, s0
995; GFX8-NEXT:    v_mov_b32_e32 v1, s1
996; GFX8-NEXT:    flat_store_dword v[0:1], v2
997; GFX8-NEXT:    s_endpgm
998;
999; GFX9-NODL-LABEL: udot2_v4i16:
1000; GFX9-NODL:       ; %bb.0: ; %entry
1001; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1002; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1003; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1004; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1005; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1006; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1007; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1008; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1009; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1010; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
1011; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1012; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
1013; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
1014; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1015; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1016; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1017; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
1018; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1019; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1020; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1021; GFX9-NODL-NEXT:    s_endpgm
1022;
1023; GFX9-DL-LABEL: udot2_v4i16:
1024; GFX9-DL:       ; %bb.0: ; %entry
1025; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1026; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1027; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1028; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1029; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
1030; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1031; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1032; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
1033; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1034; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s4, v0, v1
1035; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1036; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1037; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1038; GFX9-DL-NEXT:    s_endpgm
1039;
1040; GFX10-DL-LABEL: udot2_v4i16:
1041; GFX10-DL:       ; %bb.0: ; %entry
1042; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1043; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1044; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1045; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1046; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
1047; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1048; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1049; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1050; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
1051; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s4, s3, v0
1052; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1053; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1054; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1055; GFX10-DL-NEXT:    s_endpgm
1056                                       <4 x i16> addrspace(1)* %src2,
1057                                       i32 addrspace(1)* nocapture %dst) {
1058entry:
1059  %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
1060  %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
1061
1062  %s1.elt1 = extractelement <4 x i16> %vec1, i64 0
1063  %conv = zext i16 %s1.elt1 to i32
1064  %s2.elt1 = extractelement <4 x i16> %vec2, i64 0
1065  %conv2 = zext i16 %s2.elt1 to i32
1066  %mul1 = mul i32 %conv2, %conv
1067
1068  %s1.elt2 = extractelement <4 x i16> %vec1, i64 1
1069  %conv3 = zext i16 %s1.elt2 to i32
1070  %s2.elt2 = extractelement <4 x i16> %vec2, i64 1
1071  %conv4 = zext i16 %s2.elt2 to i32
1072  %mul2 = mul i32 %conv4, %conv3
1073
1074  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1075  %add = add i32 %mul2, %s3
1076  %add6 = add i32 %add, %mul1
1077  store i32 %add6, i32 addrspace(1)* %dst, align 4
1078  ret void
1079}
1080
1081define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1,
1082; GFX7-LABEL: udot2_v4i16_Hi:
1083; GFX7:       ; %bb.0: ; %entry
1084; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1085; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1086; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1087; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1088; GFX7-NEXT:    s_mov_b32 s2, -1
1089; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1090; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x1
1091; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x1
1092; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1093; GFX7-NEXT:    s_and_b32 s6, s4, s8
1094; GFX7-NEXT:    s_and_b32 s7, s5, s8
1095; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
1096; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
1097; GFX7-NEXT:    s_lshr_b32 s5, s5, 16
1098; GFX7-NEXT:    v_mov_b32_e32 v0, s4
1099; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1100; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1101; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1102; GFX7-NEXT:    v_mov_b32_e32 v1, s6
1103; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
1104; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1105; GFX7-NEXT:    s_endpgm
1106;
1107; GFX8-LABEL: udot2_v4i16_Hi:
1108; GFX8:       ; %bb.0: ; %entry
1109; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1110; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1111; GFX8-NEXT:    s_mov_b32 s2, 0xffff
1112; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1113; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x4
1114; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x4
1115; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
1116; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1117; GFX8-NEXT:    s_and_b32 s6, s3, s2
1118; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
1119; GFX8-NEXT:    s_and_b32 s2, s4, s2
1120; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
1121; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1122; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1123; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1124; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1125; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
1126; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1127; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1128; GFX8-NEXT:    flat_store_dword v[0:1], v2
1129; GFX8-NEXT:    s_endpgm
1130;
1131; GFX9-NODL-LABEL: udot2_v4i16_Hi:
1132; GFX9-NODL:       ; %bb.0: ; %entry
1133; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1134; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1135; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1136; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1137; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x4
1138; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x4
1139; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1140; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1141; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1142; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
1143; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1144; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
1145; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
1146; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1147; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1148; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1149; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
1150; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1151; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1152; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1153; GFX9-NODL-NEXT:    s_endpgm
1154;
1155; GFX9-DL-LABEL: udot2_v4i16_Hi:
1156; GFX9-DL:       ; %bb.0: ; %entry
1157; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1158; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1159; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1160; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x4
1161; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
1162; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x4
1163; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1164; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
1165; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1166; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s4, v0, v1
1167; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1168; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1169; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1170; GFX9-DL-NEXT:    s_endpgm
1171;
1172; GFX10-DL-LABEL: udot2_v4i16_Hi:
1173; GFX10-DL:       ; %bb.0: ; %entry
1174; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1175; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1176; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1177; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1178; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
1179; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x4
1180; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x4
1181; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1182; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
1183; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s4, s3, v0
1184; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1185; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1186; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1187; GFX10-DL-NEXT:    s_endpgm
1188                                          <4 x i16> addrspace(1)* %src2,
1189                                          i32 addrspace(1)* nocapture %dst) {
1190entry:
1191  %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
1192  %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
1193
1194  %s1.elt1 = extractelement <4 x i16> %vec1, i64 2
1195  %conv = zext i16 %s1.elt1 to i32
1196  %s2.elt1 = extractelement <4 x i16> %vec2, i64 2
1197  %conv2 = zext i16 %s2.elt1 to i32
1198  %mul1 = mul i32 %conv2, %conv
1199
1200  %s1.elt2 = extractelement <4 x i16> %vec1, i64 3
1201  %conv3 = zext i16 %s1.elt2 to i32
1202  %s2.elt2 = extractelement <4 x i16> %vec2, i64 3
1203  %conv4 = zext i16 %s2.elt2 to i32
1204  %mul2 = mul i32 %conv4, %conv3
1205
1206  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1207  %add = add i32 %mul2, %s3
1208  %add6 = add i32 %add, %mul1
1209  store i32 %add6, i32 addrspace(1)* %dst, align 4
1210  ret void
1211}
1212
1213define amdgpu_kernel void @notudot2_v4i16_Even(<4 x i16> addrspace(1)* %src1,
1214; GFX7-LABEL: notudot2_v4i16_Even:
1215; GFX7:       ; %bb.0: ; %entry
1216; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1217; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1218; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1219; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1220; GFX7-NEXT:    s_mov_b32 s2, -1
1221; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1222; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
1223; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
1224; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1225; GFX7-NEXT:    s_and_b32 s5, s5, s8
1226; GFX7-NEXT:    s_and_b32 s4, s4, s8
1227; GFX7-NEXT:    s_and_b32 s6, s6, s8
1228; GFX7-NEXT:    s_and_b32 s7, s7, s8
1229; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
1230; GFX7-NEXT:    v_mov_b32_e32 v0, s5
1231; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1232; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1233; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
1234; GFX7-NEXT:    v_mov_b32_e32 v1, s4
1235; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
1236; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1237; GFX7-NEXT:    s_endpgm
1238;
1239; GFX8-LABEL: notudot2_v4i16_Even:
1240; GFX8:       ; %bb.0: ; %entry
1241; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1242; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1243; GFX8-NEXT:    s_mov_b32 s8, 0xffff
1244; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1245; GFX8-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1246; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1247; GFX8-NEXT:    s_load_dword s6, s[0:1], 0x0
1248; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1249; GFX8-NEXT:    s_and_b32 s3, s3, s8
1250; GFX8-NEXT:    s_and_b32 s2, s2, s8
1251; GFX8-NEXT:    s_and_b32 s5, s5, s8
1252; GFX8-NEXT:    v_mov_b32_e32 v0, s6
1253; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1254; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1255; GFX8-NEXT:    s_and_b32 s4, s4, s8
1256; GFX8-NEXT:    v_mov_b32_e32 v1, s2
1257; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1258; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1259; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1260; GFX8-NEXT:    flat_store_dword v[0:1], v2
1261; GFX8-NEXT:    s_endpgm
1262;
1263; GFX9-NODL-LABEL: notudot2_v4i16_Even:
1264; GFX9-NODL:       ; %bb.0: ; %entry
1265; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1266; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1267; GFX9-NODL-NEXT:    s_mov_b32 s8, 0xffff
1268; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1269; GFX9-NODL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1270; GFX9-NODL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1271; GFX9-NODL-NEXT:    s_load_dword s6, s[0:1], 0x0
1272; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1273; GFX9-NODL-NEXT:    s_and_b32 s3, s3, s8
1274; GFX9-NODL-NEXT:    s_and_b32 s2, s2, s8
1275; GFX9-NODL-NEXT:    s_and_b32 s5, s5, s8
1276; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s6
1277; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1278; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1279; GFX9-NODL-NEXT:    s_and_b32 s4, s4, s8
1280; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
1281; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1282; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1283; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1284; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1285; GFX9-NODL-NEXT:    s_endpgm
1286;
1287; GFX9-DL-LABEL: notudot2_v4i16_Even:
1288; GFX9-DL:       ; %bb.0: ; %entry
1289; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1290; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1291; GFX9-DL-NEXT:    s_mov_b32 s8, 0xffff
1292; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1293; GFX9-DL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1294; GFX9-DL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1295; GFX9-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
1296; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1297; GFX9-DL-NEXT:    s_and_b32 s3, s3, s8
1298; GFX9-DL-NEXT:    s_and_b32 s2, s2, s8
1299; GFX9-DL-NEXT:    s_and_b32 s5, s5, s8
1300; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
1301; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1302; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1303; GFX9-DL-NEXT:    s_and_b32 s4, s4, s8
1304; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
1305; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1306; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1307; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1308; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1309; GFX9-DL-NEXT:    s_endpgm
1310;
1311; GFX10-DL-LABEL: notudot2_v4i16_Even:
1312; GFX10-DL:       ; %bb.0: ; %entry
1313; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1314; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1315; GFX10-DL-NEXT:    s_mov_b32 s8, 0xffff
1316; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1317; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1318; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1319; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1320; GFX10-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
1321; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1322; GFX10-DL-NEXT:    s_and_b32 s3, s3, s8
1323; GFX10-DL-NEXT:    s_and_b32 s5, s5, s8
1324; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1325; GFX10-DL-NEXT:    s_and_b32 s2, s2, s8
1326; GFX10-DL-NEXT:    s_and_b32 s4, s4, s8
1327; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s5, s3, v0
1328; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s4, s2, v0
1329; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1330; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1331; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1332; GFX10-DL-NEXT:    s_endpgm
1333                                               <4 x i16> addrspace(1)* %src2,
1334                                               i32 addrspace(1)* nocapture %dst) {
1335entry:
1336  %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
1337  %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
1338
1339  %s1.elt1 = extractelement <4 x i16> %vec1, i64 0
1340  %conv = zext i16 %s1.elt1 to i32
1341  %s2.elt1 = extractelement <4 x i16> %vec2, i64 0
1342  %conv2 = zext i16 %s2.elt1 to i32
1343  %mul1 = mul i32 %conv2, %conv
1344
1345  %s1.elt2 = extractelement <4 x i16> %vec1, i64 2
1346  %conv3 = zext i16 %s1.elt2 to i32
1347  %s2.elt2 = extractelement <4 x i16> %vec2, i64 2
1348  %conv4 = zext i16 %s2.elt2 to i32
1349  %mul2 = mul i32 %conv4, %conv3
1350
1351  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1352  %add = add i32 %mul2, %s3
1353  %add6 = add i32 %add, %mul1
1354  store i32 %add6, i32 addrspace(1)* %dst, align 4
1355  ret void
1356}
1357
1358define amdgpu_kernel void @notudot2_v4i16_Middle(<4 x i16> addrspace(1)* %src1,
1359; GFX7-LABEL: notudot2_v4i16_Middle:
1360; GFX7:       ; %bb.0: ; %entry
1361; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1362; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1363; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1364; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1365; GFX7-NEXT:    s_mov_b32 s2, -1
1366; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1367; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
1368; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
1369; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1370; GFX7-NEXT:    s_and_b32 s5, s5, s8
1371; GFX7-NEXT:    s_and_b32 s7, s7, s8
1372; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
1373; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
1374; GFX7-NEXT:    v_mov_b32_e32 v0, s5
1375; GFX7-NEXT:    s_lshr_b32 s6, s6, 16
1376; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1377; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1378; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
1379; GFX7-NEXT:    v_mov_b32_e32 v1, s4
1380; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
1381; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1382; GFX7-NEXT:    s_endpgm
1383;
1384; GFX8-LABEL: notudot2_v4i16_Middle:
1385; GFX8:       ; %bb.0: ; %entry
1386; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1387; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1388; GFX8-NEXT:    s_mov_b32 s8, 0xffff
1389; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1390; GFX8-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1391; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1392; GFX8-NEXT:    s_load_dword s6, s[0:1], 0x0
1393; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1394; GFX8-NEXT:    s_and_b32 s3, s3, s8
1395; GFX8-NEXT:    s_lshr_b32 s2, s2, 16
1396; GFX8-NEXT:    s_and_b32 s5, s5, s8
1397; GFX8-NEXT:    v_mov_b32_e32 v0, s6
1398; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1399; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1400; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
1401; GFX8-NEXT:    v_mov_b32_e32 v1, s2
1402; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1403; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1404; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1405; GFX8-NEXT:    flat_store_dword v[0:1], v2
1406; GFX8-NEXT:    s_endpgm
1407;
1408; GFX9-NODL-LABEL: notudot2_v4i16_Middle:
1409; GFX9-NODL:       ; %bb.0: ; %entry
1410; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1411; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1412; GFX9-NODL-NEXT:    s_mov_b32 s8, 0xffff
1413; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1414; GFX9-NODL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1415; GFX9-NODL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1416; GFX9-NODL-NEXT:    s_load_dword s6, s[0:1], 0x0
1417; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1418; GFX9-NODL-NEXT:    s_and_b32 s3, s3, s8
1419; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 16
1420; GFX9-NODL-NEXT:    s_and_b32 s5, s5, s8
1421; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s6
1422; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1423; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1424; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
1425; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
1426; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1427; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1428; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1429; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1430; GFX9-NODL-NEXT:    s_endpgm
1431;
1432; GFX9-DL-LABEL: notudot2_v4i16_Middle:
1433; GFX9-DL:       ; %bb.0: ; %entry
1434; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1435; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1436; GFX9-DL-NEXT:    s_mov_b32 s8, 0xffff
1437; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1438; GFX9-DL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1439; GFX9-DL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1440; GFX9-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
1441; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1442; GFX9-DL-NEXT:    s_and_b32 s3, s3, s8
1443; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 16
1444; GFX9-DL-NEXT:    s_and_b32 s5, s5, s8
1445; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
1446; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1447; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1448; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
1449; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
1450; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1451; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1452; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1453; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1454; GFX9-DL-NEXT:    s_endpgm
1455;
1456; GFX10-DL-LABEL: notudot2_v4i16_Middle:
1457; GFX10-DL:       ; %bb.0: ; %entry
1458; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1459; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1460; GFX10-DL-NEXT:    s_mov_b32 s8, 0xffff
1461; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1462; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1463; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1464; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1465; GFX10-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
1466; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1467; GFX10-DL-NEXT:    s_and_b32 s3, s3, s8
1468; GFX10-DL-NEXT:    s_and_b32 s5, s5, s8
1469; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1470; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 16
1471; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 16
1472; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s5, s3, v0
1473; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s4, s2, v0
1474; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1475; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1476; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1477; GFX10-DL-NEXT:    s_endpgm
1478                                                 <4 x i16> addrspace(1)* %src2,
1479                                                 i32 addrspace(1)* nocapture %dst) {
1480entry:
1481  %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
1482  %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
1483
1484  %s1.elt1 = extractelement <4 x i16> %vec1, i64 1
1485  %conv = zext i16 %s1.elt1 to i32
1486  %s2.elt1 = extractelement <4 x i16> %vec2, i64 1
1487  %conv2 = zext i16 %s2.elt1 to i32
1488  %mul1 = mul i32 %conv2, %conv
1489
1490  %s1.elt2 = extractelement <4 x i16> %vec1, i64 2
1491  %conv3 = zext i16 %s1.elt2 to i32
1492  %s2.elt2 = extractelement <4 x i16> %vec2, i64 2
1493  %conv4 = zext i16 %s2.elt2 to i32
1494  %mul2 = mul i32 %conv4, %conv3
1495
1496  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1497  %add = add i32 %mul2, %s3
1498  %add6 = add i32 %add, %mul1
1499  store i32 %add6, i32 addrspace(1)* %dst, align 4
1500  ret void
1501}
1502
1503define amdgpu_kernel void @notudot2_DiffIndex(<2 x i16> addrspace(1)* %src1,
1504; GFX7-LABEL: notudot2_DiffIndex:
1505; GFX7:       ; %bb.0: ; %entry
1506; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1507; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1508; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1509; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1510; GFX7-NEXT:    s_mov_b32 s2, -1
1511; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1512; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1513; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1514; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1515; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
1516; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
1517; GFX7-NEXT:    s_and_b32 s4, s4, s8
1518; GFX7-NEXT:    s_and_b32 s5, s5, s8
1519; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
1520; GFX7-NEXT:    v_mov_b32_e32 v0, s6
1521; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1522; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1523; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1524; GFX7-NEXT:    v_mov_b32_e32 v1, s4
1525; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
1526; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1527; GFX7-NEXT:    s_endpgm
1528;
1529; GFX8-LABEL: notudot2_DiffIndex:
1530; GFX8:       ; %bb.0: ; %entry
1531; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1532; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1533; GFX8-NEXT:    s_mov_b32 s2, 0xffff
1534; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1535; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1536; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1537; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
1538; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1539; GFX8-NEXT:    s_and_b32 s6, s3, s2
1540; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
1541; GFX8-NEXT:    s_and_b32 s2, s4, s2
1542; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1543; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1544; GFX8-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1545; GFX8-NEXT:    s_lshr_b32 s7, s4, 16
1546; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1547; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v1, v0
1548; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1549; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1550; GFX8-NEXT:    flat_store_dword v[0:1], v2
1551; GFX8-NEXT:    s_endpgm
1552;
1553; GFX9-NODL-LABEL: notudot2_DiffIndex:
1554; GFX9-NODL:       ; %bb.0: ; %entry
1555; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1556; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1557; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1558; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1559; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1560; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1561; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1562; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1563; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1564; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
1565; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1566; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
1567; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1568; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1569; GFX9-NODL-NEXT:    s_lshr_b32 s7, s4, 16
1570; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1571; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s7, v1, v0
1572; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1573; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1574; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1575; GFX9-NODL-NEXT:    s_endpgm
1576;
1577; GFX9-DL-LABEL: notudot2_DiffIndex:
1578; GFX9-DL:       ; %bb.0: ; %entry
1579; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1580; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1581; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
1582; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1583; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1584; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1585; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1586; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1587; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
1588; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
1589; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
1590; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
1591; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1592; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1593; GFX9-DL-NEXT:    s_lshr_b32 s7, s4, 16
1594; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
1595; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v1, v0
1596; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1597; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1598; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1599; GFX9-DL-NEXT:    s_endpgm
1600;
1601; GFX10-DL-LABEL: notudot2_DiffIndex:
1602; GFX10-DL:       ; %bb.0: ; %entry
1603; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1604; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1605; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
1606; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1607; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1608; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1609; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1610; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1611; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1612; GFX10-DL-NEXT:    s_lshr_b32 s6, s3, 16
1613; GFX10-DL-NEXT:    s_and_b32 s7, s4, s2
1614; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
1615; GFX10-DL-NEXT:    s_and_b32 s2, s3, s2
1616; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 16
1617; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s7, s6, v0
1618; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v0
1619; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1620; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1621; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1622; GFX10-DL-NEXT:    s_endpgm
1623                                              <2 x i16> addrspace(1)* %src2,
1624                                              i32 addrspace(1)* nocapture %dst) {
1625entry:
1626  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
1627  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
1628
1629  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
1630  %conv = zext i16 %s1.elt1 to i32
1631  %s2.elt1 = extractelement <2 x i16> %vec2, i64 1
1632  %conv2 = zext i16 %s2.elt1 to i32
1633  %mul1 = mul i32 %conv2, %conv
1634
1635  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
1636  %conv3 = zext i16 %s1.elt2 to i32
1637  %s2.elt2 = extractelement <2 x i16> %vec2, i64 0
1638  %conv4 = zext i16 %s2.elt2 to i32
1639  %mul2 = mul i32 %conv4, %conv3
1640
1641  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1642  %add = add i32 %mul2, %s3
1643  %add6 = add i32 %add, %mul1
1644  store i32 %add6, i32 addrspace(1)* %dst, align 4
1645  ret void
1646}
1647
1648define amdgpu_kernel void @udot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1,
1649; GFX7-LABEL: udot2_MultipleUses_add1:
1650; GFX7:       ; %bb.0: ; %entry
1651; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1652; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1653; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1654; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1655; GFX7-NEXT:    s_mov_b32 s2, -1
1656; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1657; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1658; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1659; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1660; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
1661; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
1662; GFX7-NEXT:    s_and_b32 s4, s4, s8
1663; GFX7-NEXT:    s_and_b32 s5, s5, s8
1664; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
1665; GFX7-NEXT:    v_mov_b32_e32 v0, s6
1666; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1667; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1668; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
1669; GFX7-NEXT:    v_mov_b32_e32 v1, s4
1670; GFX7-NEXT:    v_mad_u32_u24 v1, s5, v1, v0
1671; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
1672; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1673; GFX7-NEXT:    s_endpgm
1674;
1675; GFX8-LABEL: udot2_MultipleUses_add1:
1676; GFX8:       ; %bb.0: ; %entry
1677; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1678; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1679; GFX8-NEXT:    s_mov_b32 s2, 0xffff
1680; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1681; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1682; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1683; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
1684; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1685; GFX8-NEXT:    s_and_b32 s6, s3, s2
1686; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
1687; GFX8-NEXT:    s_and_b32 s2, s4, s2
1688; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
1689; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1690; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1691; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1692; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1693; GFX8-NEXT:    v_mad_u32_u24 v1, s2, v1, v0
1694; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v1
1695; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1696; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1697; GFX8-NEXT:    flat_store_dword v[0:1], v2
1698; GFX8-NEXT:    s_endpgm
1699;
1700; GFX9-NODL-LABEL: udot2_MultipleUses_add1:
1701; GFX9-NODL:       ; %bb.0: ; %entry
1702; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1703; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1704; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1705; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1706; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1707; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1708; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1709; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1710; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1711; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
1712; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1713; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
1714; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
1715; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1716; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1717; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1718; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s2, v1, v0
1719; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v1, v0
1720; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1721; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1722; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1723; GFX9-NODL-NEXT:    s_endpgm
1724;
1725; GFX9-DL-LABEL: udot2_MultipleUses_add1:
1726; GFX9-DL:       ; %bb.0: ; %entry
1727; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1728; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1729; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
1730; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1731; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1732; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1733; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1734; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1735; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
1736; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
1737; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
1738; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
1739; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
1740; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1741; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1742; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
1743; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s2, v1, v0
1744; GFX9-DL-NEXT:    v_add_u32_e32 v2, v1, v0
1745; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1746; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1747; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1748; GFX9-DL-NEXT:    s_endpgm
1749;
1750; GFX10-DL-LABEL: udot2_MultipleUses_add1:
1751; GFX10-DL:       ; %bb.0: ; %entry
1752; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1753; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1754; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
1755; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1756; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1757; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1758; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1759; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1760; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1761; GFX10-DL-NEXT:    s_lshr_b32 s6, s3, 16
1762; GFX10-DL-NEXT:    s_lshr_b32 s7, s4, 16
1763; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
1764; GFX10-DL-NEXT:    s_and_b32 s3, s3, s2
1765; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
1766; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s7, s6, v0
1767; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s2, s3, v0
1768; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v1, v0
1769; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1770; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1771; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1772; GFX10-DL-NEXT:    s_endpgm
1773                                                   <2 x i16> addrspace(1)* %src2,
1774                                                   i32 addrspace(1)* nocapture %dst) {
1775entry:
1776  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
1777  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
1778
1779  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
1780  %conv = zext i16 %s1.elt1 to i32
1781  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
1782  %conv2 = zext i16 %s2.elt1 to i32
1783  %mul1 = mul i32 %conv2, %conv
1784
1785  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
1786  %conv3 = zext i16 %s1.elt2 to i32
1787  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
1788  %conv4 = zext i16 %s2.elt2 to i32
1789  %mul2 = mul i32 %conv4, %conv3
1790
1791  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1792  %add1 = add i32 %mul2, %s3
1793  %add2 = add i32 %add1, %mul1
1794
1795  %res = add i32 %add2, %add1
1796  store i32 %res, i32 addrspace(1)* %dst, align 4
1797  ret void
1798}
1799
1800define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1,
1801; GFX7-LABEL: idot2_MultipleUses_add1:
1802; GFX7:       ; %bb.0: ; %entry
1803; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1804; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1805; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1806; GFX7-NEXT:    s_mov_b32 s2, -1
1807; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1808; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1809; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1810; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
1811; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1812; GFX7-NEXT:    s_sext_i32_i16 s7, s4
1813; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
1814; GFX7-NEXT:    s_sext_i32_i16 s8, s5
1815; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
1816; GFX7-NEXT:    v_mov_b32_e32 v0, s4
1817; GFX7-NEXT:    v_mov_b32_e32 v1, s6
1818; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
1819; GFX7-NEXT:    v_mov_b32_e32 v1, s7
1820; GFX7-NEXT:    v_mad_i32_i24 v1, s8, v1, v0
1821; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
1822; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1823; GFX7-NEXT:    s_endpgm
1824;
1825; GFX8-LABEL: idot2_MultipleUses_add1:
1826; GFX8:       ; %bb.0: ; %entry
1827; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1828; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1829; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1830; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
1831; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
1832; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
1833; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1834; GFX8-NEXT:    s_sext_i32_i16 s5, s2
1835; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
1836; GFX8-NEXT:    s_sext_i32_i16 s6, s3
1837; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
1838; GFX8-NEXT:    v_mov_b32_e32 v0, s4
1839; GFX8-NEXT:    v_mov_b32_e32 v1, s2
1840; GFX8-NEXT:    v_mov_b32_e32 v2, s5
1841; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
1842; GFX8-NEXT:    v_mad_i32_i24 v1, s6, v2, v0
1843; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v1
1844; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1845; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1846; GFX8-NEXT:    flat_store_dword v[0:1], v2
1847; GFX8-NEXT:    s_endpgm
1848;
1849; GFX9-NODL-LABEL: idot2_MultipleUses_add1:
1850; GFX9-NODL:       ; %bb.0: ; %entry
1851; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1852; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1853; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1854; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
1855; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
1856; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
1857; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1858; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
1859; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
1860; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
1861; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
1862; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
1863; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
1864; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
1865; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
1866; GFX9-NODL-NEXT:    v_mad_i32_i24 v1, s6, v2, v0
1867; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v1, v0
1868; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1869; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1870; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1871; GFX9-NODL-NEXT:    s_endpgm
1872;
1873; GFX9-DL-LABEL: idot2_MultipleUses_add1:
1874; GFX9-DL:       ; %bb.0: ; %entry
1875; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1876; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1877; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1878; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1879; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1880; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
1881; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1882; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
1883; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
1884; GFX9-DL-NEXT:    s_sext_i32_i16 s6, s3
1885; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
1886; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
1887; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
1888; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
1889; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
1890; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s6, v2, v0
1891; GFX9-DL-NEXT:    v_add_u32_e32 v2, v1, v0
1892; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1893; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1894; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1895; GFX9-DL-NEXT:    s_endpgm
1896;
1897; GFX10-DL-LABEL: idot2_MultipleUses_add1:
1898; GFX10-DL:       ; %bb.0: ; %entry
1899; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1900; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1901; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1902; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1903; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1904; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1905; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
1906; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1907; GFX10-DL-NEXT:    s_ashr_i32 s5, s2, 16
1908; GFX10-DL-NEXT:    s_ashr_i32 s6, s3, 16
1909; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
1910; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s2
1911; GFX10-DL-NEXT:    s_sext_i32_i16 s3, s3
1912; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s5, v0
1913; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s3, s2, v0
1914; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v1, v0
1915; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1916; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1917; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1918; GFX10-DL-NEXT:    s_endpgm
1919                                                   <2 x i16> addrspace(1)* %src2,
1920                                                   i32 addrspace(1)* nocapture %dst) {
1921entry:
1922  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
1923  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
1924
1925  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
1926  %conv = sext i16 %s1.elt1 to i32
1927  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
1928  %conv2 = sext i16 %s2.elt1 to i32
1929  %mul1 = mul i32 %conv2, %conv
1930
1931  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
1932  %conv3 = sext i16 %s1.elt2 to i32
1933  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
1934  %conv4 = sext i16 %s2.elt2 to i32
1935  %mul2 = mul i32 %conv4, %conv3
1936
1937  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1938  %add1 = add i32 %mul2, %s3
1939  %add2 = add i32 %add1, %mul1
1940
1941  %res = add i32 %add2, %add1
1942  store i32 %res, i32 addrspace(1)* %dst, align 4
1943  ret void
1944}
1945
1946define amdgpu_kernel void @udot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1,
1947; GFX7-LABEL: udot2_MultipleUses_mul1:
1948; GFX7:       ; %bb.0: ; %entry
1949; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1950; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1951; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1952; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1953; GFX7-NEXT:    s_mov_b32 s2, -1
1954; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1955; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1956; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1957; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1958; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
1959; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
1960; GFX7-NEXT:    s_and_b32 s4, s4, s8
1961; GFX7-NEXT:    s_and_b32 s5, s5, s8
1962; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
1963; GFX7-NEXT:    v_mov_b32_e32 v0, s4
1964; GFX7-NEXT:    v_mov_b32_e32 v2, s6
1965; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1966; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1967; GFX7-NEXT:    v_mad_u32_u24 v1, s5, v0, v1
1968; GFX7-NEXT:    v_mad_u32_u24 v1, s7, v2, v1
1969; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1970; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1971; GFX7-NEXT:    s_endpgm
1972;
1973; GFX8-LABEL: udot2_MultipleUses_mul1:
1974; GFX8:       ; %bb.0: ; %entry
1975; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1976; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1977; GFX8-NEXT:    s_mov_b32 s2, 0xffff
1978; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1979; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1980; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1981; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
1982; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1983; GFX8-NEXT:    s_and_b32 s6, s3, s2
1984; GFX8-NEXT:    s_and_b32 s2, s4, s2
1985; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
1986; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1987; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1988; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
1989; GFX8-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1990; GFX8-NEXT:    v_mov_b32_e32 v2, s3
1991; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v2, v0
1992; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
1993; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1994; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1995; GFX8-NEXT:    flat_store_dword v[0:1], v2
1996; GFX8-NEXT:    s_endpgm
1997;
1998; GFX9-NODL-LABEL: udot2_MultipleUses_mul1:
1999; GFX9-NODL:       ; %bb.0: ; %entry
2000; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2001; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2002; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
2003; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2004; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
2005; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
2006; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
2007; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2008; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
2009; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
2010; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
2011; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
2012; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
2013; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
2014; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
2015; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s3
2016; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v2, v0
2017; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2018; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2019; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2020; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2021; GFX9-NODL-NEXT:    s_endpgm
2022;
2023; GFX9-DL-LABEL: udot2_MultipleUses_mul1:
2024; GFX9-DL:       ; %bb.0: ; %entry
2025; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2026; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2027; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
2028; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2029; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
2030; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
2031; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
2032; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2033; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
2034; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
2035; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
2036; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
2037; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
2038; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
2039; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
2040; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
2041; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v2, v0
2042; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2043; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2044; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2045; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2046; GFX9-DL-NEXT:    s_endpgm
2047;
2048; GFX10-DL-LABEL: udot2_MultipleUses_mul1:
2049; GFX10-DL:       ; %bb.0: ; %entry
2050; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2051; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2052; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
2053; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2054; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2055; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
2056; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
2057; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
2058; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2059; GFX10-DL-NEXT:    s_and_b32 s6, s3, s2
2060; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
2061; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
2062; GFX10-DL-NEXT:    s_lshr_b32 s3, s3, 16
2063; GFX10-DL-NEXT:    s_lshr_b32 s4, s4, 16
2064; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s2, s6, v0
2065; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s4, s3, v0
2066; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s6, v0
2067; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2068; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2069; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2070; GFX10-DL-NEXT:    s_endpgm
2071                                                   <2 x i16> addrspace(1)* %src2,
2072                                                   i32 addrspace(1)* nocapture %dst) {
2073entry:
2074  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2075  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2076
2077  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
2078  %conv = zext i16 %s1.elt1 to i32
2079  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
2080  %conv2 = zext i16 %s2.elt1 to i32
2081  %mul1 = mul i32 %conv2, %conv
2082
2083  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
2084  %conv3 = zext i16 %s1.elt2 to i32
2085  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
2086  %conv4 = zext i16 %s2.elt2 to i32
2087  %mul2 = mul i32 %conv4, %conv3
2088
2089  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2090  %add0 = add i32 %mul1, %s3
2091
2092  %add1 = add i32 %mul2, %add0
2093  %add2 = add i32 %add1, %mul1
2094
2095  store i32 %add2, i32 addrspace(1)* %dst, align 4
2096  ret void
2097}
2098
2099define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1,
2100; GFX7-LABEL: idot2_MultipleUses_mul1:
2101; GFX7:       ; %bb.0: ; %entry
2102; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2103; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2104; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2105; GFX7-NEXT:    s_mov_b32 s2, -1
2106; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2107; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2108; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2109; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
2110; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2111; GFX7-NEXT:    s_sext_i32_i16 s7, s4
2112; GFX7-NEXT:    s_sext_i32_i16 s8, s5
2113; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
2114; GFX7-NEXT:    v_mov_b32_e32 v0, s7
2115; GFX7-NEXT:    v_mov_b32_e32 v1, s6
2116; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
2117; GFX7-NEXT:    v_mad_i32_i24 v1, s8, v0, v1
2118; GFX7-NEXT:    v_mov_b32_e32 v2, s4
2119; GFX7-NEXT:    v_mad_i32_i24 v1, s5, v2, v1
2120; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v0, v1
2121; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2122; GFX7-NEXT:    s_endpgm
2123;
2124; GFX8-LABEL: idot2_MultipleUses_mul1:
2125; GFX8:       ; %bb.0: ; %entry
2126; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2127; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2128; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2129; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
2130; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
2131; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
2132; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2133; GFX8-NEXT:    s_sext_i32_i16 s5, s2
2134; GFX8-NEXT:    s_sext_i32_i16 s6, s3
2135; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
2136; GFX8-NEXT:    v_mov_b32_e32 v0, s4
2137; GFX8-NEXT:    v_mov_b32_e32 v1, s5
2138; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
2139; GFX8-NEXT:    v_mov_b32_e32 v2, s2
2140; GFX8-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
2141; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v2, v0
2142; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v1, v0
2143; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2144; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2145; GFX8-NEXT:    flat_store_dword v[0:1], v2
2146; GFX8-NEXT:    s_endpgm
2147;
2148; GFX9-NODL-LABEL: idot2_MultipleUses_mul1:
2149; GFX9-NODL:       ; %bb.0: ; %entry
2150; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2151; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2152; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2153; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
2154; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
2155; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
2156; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2157; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
2158; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
2159; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
2160; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
2161; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s5
2162; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
2163; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s2
2164; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
2165; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v2, v0
2166; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v1, v0
2167; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2168; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2169; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2170; GFX9-NODL-NEXT:    s_endpgm
2171;
2172; GFX9-DL-LABEL: idot2_MultipleUses_mul1:
2173; GFX9-DL:       ; %bb.0: ; %entry
2174; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2175; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2176; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2177; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2178; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2179; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
2180; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2181; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
2182; GFX9-DL-NEXT:    s_sext_i32_i16 s6, s3
2183; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
2184; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
2185; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s5
2186; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
2187; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
2188; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
2189; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v2, v0
2190; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v1, v0
2191; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2192; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2193; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2194; GFX9-DL-NEXT:    s_endpgm
2195;
2196; GFX10-DL-LABEL: idot2_MultipleUses_mul1:
2197; GFX10-DL:       ; %bb.0: ; %entry
2198; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2199; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2200; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2201; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2202; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2203; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2204; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
2205; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2206; GFX10-DL-NEXT:    s_sext_i32_i16 s5, s2
2207; GFX10-DL-NEXT:    s_sext_i32_i16 s6, s3
2208; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
2209; GFX10-DL-NEXT:    s_ashr_i32 s2, s2, 16
2210; GFX10-DL-NEXT:    s_ashr_i32 s3, s3, 16
2211; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s5, v0
2212; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s3, s2, v0
2213; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s6, s5, v0
2214; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2215; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2216; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2217; GFX10-DL-NEXT:    s_endpgm
2218                                                   <2 x i16> addrspace(1)* %src2,
2219                                                   i32 addrspace(1)* nocapture %dst) {
2220entry:
2221  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2222  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2223
2224  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
2225  %conv = sext i16 %s1.elt1 to i32
2226  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
2227  %conv2 = sext i16 %s2.elt1 to i32
2228  %mul1 = mul i32 %conv2, %conv
2229
2230  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
2231  %conv3 = sext i16 %s1.elt2 to i32
2232  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
2233  %conv4 = sext i16 %s2.elt2 to i32
2234  %mul2 = mul i32 %conv4, %conv3
2235
2236  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2237  %add0 = add i32 %mul1, %s3
2238
2239  %add1 = add i32 %mul2, %add0
2240  %add2 = add i32 %add1, %mul1
2241
2242  store i32 %add2, i32 addrspace(1)* %dst, align 4
2243  ret void
2244}
2245
2246define amdgpu_kernel void @udot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1,
2247; GFX7-LABEL: udot2_MultipleUses_mul2:
2248; GFX7:       ; %bb.0: ; %entry
2249; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2250; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2251; GFX7-NEXT:    s_mov_b32 s8, 0xffff
2252; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2253; GFX7-NEXT:    s_mov_b32 s2, -1
2254; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2255; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2256; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2257; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2258; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
2259; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
2260; GFX7-NEXT:    s_and_b32 s4, s4, s8
2261; GFX7-NEXT:    s_and_b32 s5, s5, s8
2262; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
2263; GFX7-NEXT:    v_mov_b32_e32 v0, s6
2264; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2265; GFX7-NEXT:    v_mov_b32_e32 v1, s8
2266; GFX7-NEXT:    v_mad_u32_u24 v1, s7, v0, v1
2267; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
2268; GFX7-NEXT:    v_mov_b32_e32 v1, s4
2269; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
2270; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2271; GFX7-NEXT:    s_endpgm
2272;
2273; GFX8-LABEL: udot2_MultipleUses_mul2:
2274; GFX8:       ; %bb.0: ; %entry
2275; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2276; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2277; GFX8-NEXT:    s_mov_b32 s2, 0xffff
2278; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2279; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
2280; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
2281; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
2282; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2283; GFX8-NEXT:    s_and_b32 s6, s3, s2
2284; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
2285; GFX8-NEXT:    s_and_b32 s2, s4, s2
2286; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
2287; GFX8-NEXT:    v_mov_b32_e32 v0, s5
2288; GFX8-NEXT:    v_mov_b32_e32 v1, s3
2289; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2290; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2291; GFX8-NEXT:    v_mov_b32_e32 v1, s6
2292; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2293; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2294; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2295; GFX8-NEXT:    flat_store_dword v[0:1], v2
2296; GFX8-NEXT:    s_endpgm
2297;
2298; GFX9-NODL-LABEL: udot2_MultipleUses_mul2:
2299; GFX9-NODL:       ; %bb.0: ; %entry
2300; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2301; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2302; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
2303; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2304; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
2305; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
2306; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
2307; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2308; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
2309; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
2310; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
2311; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
2312; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
2313; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
2314; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2315; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2316; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
2317; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2318; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2319; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2320; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2321; GFX9-NODL-NEXT:    s_endpgm
2322;
2323; GFX9-DL-LABEL: udot2_MultipleUses_mul2:
2324; GFX9-DL:       ; %bb.0: ; %entry
2325; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2326; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2327; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
2328; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2329; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
2330; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
2331; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
2332; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2333; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
2334; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
2335; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
2336; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
2337; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
2338; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
2339; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2340; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2341; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
2342; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2343; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2344; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2345; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2346; GFX9-DL-NEXT:    s_endpgm
2347;
2348; GFX10-DL-LABEL: udot2_MultipleUses_mul2:
2349; GFX10-DL:       ; %bb.0: ; %entry
2350; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2351; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2352; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
2353; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2354; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2355; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
2356; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
2357; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
2358; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2359; GFX10-DL-NEXT:    s_lshr_b32 s6, s3, 16
2360; GFX10-DL-NEXT:    s_lshr_b32 s7, s4, 16
2361; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s5
2362; GFX10-DL-NEXT:    s_and_b32 s3, s3, s2
2363; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
2364; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s7, s6, v0
2365; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s7, s6, v0
2366; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v0
2367; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2368; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2369; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2370; GFX10-DL-NEXT:    s_endpgm
2371                                                   <2 x i16> addrspace(1)* %src2,
2372                                                   i32 addrspace(1)* nocapture %dst) {
2373entry:
2374  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2375  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2376
2377  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
2378  %conv = zext i16 %s1.elt1 to i32
2379  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
2380  %conv2 = zext i16 %s2.elt1 to i32
2381  %mul1 = mul i32 %conv2, %conv
2382
2383  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
2384  %conv3 = zext i16 %s1.elt2 to i32
2385  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
2386  %conv4 = zext i16 %s2.elt2 to i32
2387  %mul2 = mul i32 %conv4, %conv3
2388
2389  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2390  %add0 = add i32 %mul2, %s3
2391
2392  %add1 = add i32 %mul2, %add0
2393  %add2 = add i32 %add1, %mul1
2394
2395  store i32 %add2, i32 addrspace(1)* %dst, align 4
2396  ret void
2397}
2398
2399define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1,
2400; GFX7-LABEL: idot2_MultipleUses_mul2:
2401; GFX7:       ; %bb.0: ; %entry
2402; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2403; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2404; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2405; GFX7-NEXT:    s_mov_b32 s2, -1
2406; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2407; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2408; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2409; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
2410; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2411; GFX7-NEXT:    s_sext_i32_i16 s7, s4
2412; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
2413; GFX7-NEXT:    s_sext_i32_i16 s8, s5
2414; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
2415; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2416; GFX7-NEXT:    v_mov_b32_e32 v1, s6
2417; GFX7-NEXT:    v_mad_i32_i24 v1, s5, v0, v1
2418; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
2419; GFX7-NEXT:    v_mov_b32_e32 v1, s7
2420; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
2421; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2422; GFX7-NEXT:    s_endpgm
2423;
2424; GFX8-LABEL: idot2_MultipleUses_mul2:
2425; GFX8:       ; %bb.0: ; %entry
2426; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2427; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2428; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2429; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
2430; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
2431; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
2432; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2433; GFX8-NEXT:    s_sext_i32_i16 s5, s2
2434; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
2435; GFX8-NEXT:    s_sext_i32_i16 s6, s3
2436; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
2437; GFX8-NEXT:    v_mov_b32_e32 v0, s4
2438; GFX8-NEXT:    v_mov_b32_e32 v1, s2
2439; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
2440; GFX8-NEXT:    v_mov_b32_e32 v2, s5
2441; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
2442; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
2443; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2444; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2445; GFX8-NEXT:    flat_store_dword v[0:1], v2
2446; GFX8-NEXT:    s_endpgm
2447;
2448; GFX9-NODL-LABEL: idot2_MultipleUses_mul2:
2449; GFX9-NODL:       ; %bb.0: ; %entry
2450; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2451; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2452; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2453; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
2454; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
2455; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
2456; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2457; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
2458; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
2459; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
2460; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
2461; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
2462; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
2463; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
2464; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
2465; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
2466; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
2467; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2468; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2469; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2470; GFX9-NODL-NEXT:    s_endpgm
2471;
2472; GFX9-DL-LABEL: idot2_MultipleUses_mul2:
2473; GFX9-DL:       ; %bb.0: ; %entry
2474; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2475; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2476; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2477; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2478; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2479; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
2480; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2481; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
2482; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
2483; GFX9-DL-NEXT:    s_sext_i32_i16 s6, s3
2484; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
2485; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
2486; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
2487; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
2488; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
2489; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
2490; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
2491; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2492; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2493; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2494; GFX9-DL-NEXT:    s_endpgm
2495;
2496; GFX10-DL-LABEL: idot2_MultipleUses_mul2:
2497; GFX10-DL:       ; %bb.0: ; %entry
2498; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2499; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2500; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2501; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2502; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2503; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2504; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
2505; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2506; GFX10-DL-NEXT:    s_ashr_i32 s5, s2, 16
2507; GFX10-DL-NEXT:    s_ashr_i32 s6, s3, 16
2508; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
2509; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s2
2510; GFX10-DL-NEXT:    s_sext_i32_i16 s3, s3
2511; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s5, v0
2512; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s6, s5, v0
2513; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v0
2514; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2515; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2516; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2517; GFX10-DL-NEXT:    s_endpgm
2518                                                   <2 x i16> addrspace(1)* %src2,
2519                                                   i32 addrspace(1)* nocapture %dst) {
2520entry:
2521  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2522  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2523
2524  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
2525  %conv = sext i16 %s1.elt1 to i32
2526  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
2527  %conv2 = sext i16 %s2.elt1 to i32
2528  %mul1 = mul i32 %conv2, %conv
2529
2530  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
2531  %conv3 = sext i16 %s1.elt2 to i32
2532  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
2533  %conv4 = sext i16 %s2.elt2 to i32
2534  %mul2 = mul i32 %conv4, %conv3
2535
2536  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2537  %add0 = add i32 %mul2, %s3
2538
2539  %add1 = add i32 %mul2, %add0
2540  %add2 = add i32 %add1, %mul1
2541
2542  store i32 %add2, i32 addrspace(1)* %dst, align 4
2543  ret void
2544}
2545
2546define amdgpu_kernel void @udot2_acc16(<2 x i16> addrspace(1)* %src1,
2547; GFX7-LABEL: udot2_acc16:
2548; GFX7:       ; %bb.0: ; %entry
2549; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2550; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2551; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2552; GFX7-NEXT:    s_mov_b32 s2, -1
2553; GFX7-NEXT:    s_mov_b32 s8, 0xffff
2554; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2555; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
2556; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2557; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2558; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2559; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
2560; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
2561; GFX7-NEXT:    v_mov_b32_e32 v1, s7
2562; GFX7-NEXT:    s_and_b32 s5, s5, s8
2563; GFX7-NEXT:    s_and_b32 s4, s4, s8
2564; GFX7-NEXT:    s_waitcnt vmcnt(0)
2565; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
2566; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2567; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2568; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
2569; GFX7-NEXT:    s_endpgm
2570;
2571; GFX8-LABEL: udot2_acc16:
2572; GFX8:       ; %bb.0: ; %entry
2573; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2574; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2575; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2576; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
2577; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2578; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2579; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
2580; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
2581; GFX8-NEXT:    s_mov_b32 s0, 0xffff
2582; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2583; GFX8-NEXT:    s_and_b32 s3, s1, s0
2584; GFX8-NEXT:    s_lshr_b32 s1, s1, 16
2585; GFX8-NEXT:    s_and_b32 s0, s2, s0
2586; GFX8-NEXT:    s_lshr_b32 s2, s2, 16
2587; GFX8-NEXT:    v_mov_b32_e32 v3, s1
2588; GFX8-NEXT:    s_waitcnt vmcnt(0)
2589; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
2590; GFX8-NEXT:    v_mov_b32_e32 v3, s3
2591; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
2592; GFX8-NEXT:    flat_store_short v[0:1], v2
2593; GFX8-NEXT:    s_endpgm
2594;
2595; GFX9-NODL-LABEL: udot2_acc16:
2596; GFX9-NODL:       ; %bb.0: ; %entry
2597; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2598; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2599; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2600; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
2601; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2602; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2603; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
2604; GFX9-NODL-NEXT:    s_load_dword s1, s[6:7], 0x0
2605; GFX9-NODL-NEXT:    s_mov_b32 s0, 0xffff
2606; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2607; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s0
2608; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 16
2609; GFX9-NODL-NEXT:    s_and_b32 s0, s2, s0
2610; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 16
2611; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
2612; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
2613; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
2614; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
2615; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
2616; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
2617; GFX9-NODL-NEXT:    s_endpgm
2618;
2619; GFX9-DL-LABEL: udot2_acc16:
2620; GFX9-DL:       ; %bb.0: ; %entry
2621; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2622; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2623; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2624; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
2625; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2626; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2627; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
2628; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
2629; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2630; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
2631; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2632; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s0, v3, v2
2633; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
2634; GFX9-DL-NEXT:    s_endpgm
2635;
2636; GFX10-DL-LABEL: udot2_acc16:
2637; GFX10-DL:       ; %bb.0: ; %entry
2638; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
2639; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2640; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2641; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
2642; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s3
2643; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2644; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
2645; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2646; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
2647; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
2648; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2649; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s0, s1, v2
2650; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
2651; GFX10-DL-NEXT:    s_endpgm
2652                                       <2 x i16> addrspace(1)* %src2,
2653                                       i16 addrspace(1)* nocapture %dst) {
2654entry:
2655  %v1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2656  %v2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2657
2658  %v1e1 = extractelement <2 x i16> %v1, i64 0
2659  %v2e1 = extractelement <2 x i16> %v2, i64 0
2660  %mul1 = mul i16 %v1e1, %v2e1
2661
2662  %v1e2 = extractelement <2 x i16> %v1, i64 1
2663  %v2e2 = extractelement <2 x i16> %v2, i64 1
2664  %mul2 = mul i16 %v1e2, %v2e2
2665
2666  %s2 = load i16, i16 addrspace(1)* %dst, align 2
2667  %add1 = add i16 %mul2, %s2
2668  %add2 = add i16 %add1, %mul1
2669  store i16 %add2, i16 addrspace(1)* %dst, align 2
2670  ret void
2671}
2672
2673define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
2674; GFX7-LABEL: notsdot2_sext8:
2675; GFX7:       ; %bb.0: ; %entry
2676; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2677; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2678; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2679; GFX7-NEXT:    s_mov_b32 s2, -1
2680; GFX7-NEXT:    s_mov_b32 s10, s2
2681; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2682; GFX7-NEXT:    s_mov_b32 s8, s6
2683; GFX7-NEXT:    s_mov_b32 s9, s7
2684; GFX7-NEXT:    s_mov_b32 s6, s2
2685; GFX7-NEXT:    s_mov_b32 s7, s3
2686; GFX7-NEXT:    s_mov_b32 s11, s3
2687; GFX7-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
2688; GFX7-NEXT:    buffer_load_ushort v1, off, s[8:11], 0
2689; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
2690; GFX7-NEXT:    s_waitcnt vmcnt(1)
2691; GFX7-NEXT:    v_bfe_i32 v2, v0, 0, 8
2692; GFX7-NEXT:    s_waitcnt vmcnt(0)
2693; GFX7-NEXT:    v_bfe_i32 v3, v1, 0, 8
2694; GFX7-NEXT:    v_bfe_i32 v0, v0, 8, 8
2695; GFX7-NEXT:    v_bfe_i32 v1, v1, 8, 8
2696; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2697; GFX7-NEXT:    v_mad_i32_i24 v0, v1, v0, s4
2698; GFX7-NEXT:    v_mad_i32_i24 v0, v3, v2, v0
2699; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2700; GFX7-NEXT:    s_endpgm
2701;
2702; GFX8-LABEL: notsdot2_sext8:
2703; GFX8:       ; %bb.0: ; %entry
2704; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2705; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2706; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2707; GFX8-NEXT:    v_mov_b32_e32 v0, s6
2708; GFX8-NEXT:    v_mov_b32_e32 v2, s4
2709; GFX8-NEXT:    v_mov_b32_e32 v3, s5
2710; GFX8-NEXT:    s_load_dword s2, s[0:1], 0x0
2711; GFX8-NEXT:    v_mov_b32_e32 v1, s7
2712; GFX8-NEXT:    flat_load_ushort v2, v[2:3]
2713; GFX8-NEXT:    flat_load_ushort v0, v[0:1]
2714; GFX8-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(0)
2715; GFX8-NEXT:    v_bfe_i32 v1, v2, 0, 8
2716; GFX8-NEXT:    v_lshrrev_b16_e32 v2, 8, v2
2717; GFX8-NEXT:    s_waitcnt vmcnt(0)
2718; GFX8-NEXT:    v_bfe_i32 v3, v0, 0, 8
2719; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
2720; GFX8-NEXT:    v_bfe_i32 v2, v2, 0, 8
2721; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 8
2722; GFX8-NEXT:    v_mad_i32_i24 v0, v0, v2, s2
2723; GFX8-NEXT:    v_mad_i32_i24 v2, v3, v1, v0
2724; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2725; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2726; GFX8-NEXT:    flat_store_dword v[0:1], v2
2727; GFX8-NEXT:    s_endpgm
2728;
2729; GFX9-NODL-LABEL: notsdot2_sext8:
2730; GFX9-NODL:       ; %bb.0: ; %entry
2731; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2732; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2733; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2734; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s6
2735; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
2736; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s5
2737; GFX9-NODL-NEXT:    s_load_dword s2, s[0:1], 0x0
2738; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s7
2739; GFX9-NODL-NEXT:    global_load_ushort v2, v[2:3], off
2740; GFX9-NODL-NEXT:    global_load_ushort v0, v[0:1], off
2741; GFX9-NODL-NEXT:    s_waitcnt vmcnt(1)
2742; GFX9-NODL-NEXT:    v_bfe_i32 v1, v2, 0, 8
2743; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v2, 8, v2
2744; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
2745; GFX9-NODL-NEXT:    v_bfe_i32 v3, v0, 0, 8
2746; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
2747; GFX9-NODL-NEXT:    v_bfe_i32 v2, v2, 0, 8
2748; GFX9-NODL-NEXT:    v_bfe_i32 v0, v0, 0, 8
2749; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2750; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, v0, v2, s2
2751; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, v3, v1, v0
2752; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2753; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2754; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2755; GFX9-NODL-NEXT:    s_endpgm
2756;
2757; GFX9-DL-LABEL: notsdot2_sext8:
2758; GFX9-DL:       ; %bb.0: ; %entry
2759; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2760; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2761; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2762; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
2763; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
2764; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
2765; GFX9-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
2766; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s7
2767; GFX9-DL-NEXT:    global_load_ushort v2, v[2:3], off
2768; GFX9-DL-NEXT:    global_load_ushort v0, v[0:1], off
2769; GFX9-DL-NEXT:    s_waitcnt vmcnt(1)
2770; GFX9-DL-NEXT:    v_bfe_i32 v1, v2, 0, 8
2771; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v2, 8, v2
2772; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2773; GFX9-DL-NEXT:    v_bfe_i32 v3, v0, 0, 8
2774; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
2775; GFX9-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
2776; GFX9-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
2777; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2778; GFX9-DL-NEXT:    v_mad_i32_i24 v0, v0, v2, s2
2779; GFX9-DL-NEXT:    v_mad_i32_i24 v2, v3, v1, v0
2780; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2781; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2782; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2783; GFX9-DL-NEXT:    s_endpgm
2784;
2785; GFX10-DL-LABEL: notsdot2_sext8:
2786; GFX10-DL:       ; %bb.0: ; %entry
2787; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2788; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2789; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2790; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2791; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
2792; GFX10-DL-NEXT:    v_mov_b32_e32 v3, s5
2793; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
2794; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s7
2795; GFX10-DL-NEXT:    global_load_ushort v2, v[2:3], off
2796; GFX10-DL-NEXT:    global_load_ushort v0, v[0:1], off
2797; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
2798; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
2799; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v1, 8, v2
2800; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
2801; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, v0
2802; GFX10-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
2803; GFX10-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
2804; GFX10-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
2805; GFX10-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
2806; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2807; GFX10-DL-NEXT:    v_mad_i32_i24 v1, v3, v1, s2
2808; GFX10-DL-NEXT:    v_mad_i32_i24 v2, v0, v2, v1
2809; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2810; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2811; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2812; GFX10-DL-NEXT:    s_endpgm
2813                                          <2 x i8> addrspace(1)* %src2,
2814                                          i32 addrspace(1)* nocapture %dst) {
2815entry:
2816  %vec1 = load <2 x i8>, <2 x i8> addrspace(1)* %src1
2817  %vec2 = load <2 x i8>, <2 x i8> addrspace(1)* %src2
2818
2819  %s1.elt1 = extractelement <2 x i8> %vec1, i64 0
2820  %conv = sext i8 %s1.elt1 to i32
2821  %s2.elt1 = extractelement <2 x i8> %vec2, i64 0
2822  %conv2 = sext i8 %s2.elt1 to i32
2823  %mul1 = mul nuw i32 %conv2, %conv
2824
2825  %s1.elt2 = extractelement <2 x i8> %vec1, i64 1
2826  %conv3 = sext i8 %s1.elt2 to i32
2827  %s2.elt2 = extractelement <2 x i8> %vec2, i64 1
2828  %conv4 = sext i8 %s2.elt2 to i32
2829  %mul2 = mul nuw i32 %conv4, %conv3
2830
2831  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2832  %add = add i32 %mul2, %s3
2833  %add6 = add i32 %add, %mul1
2834  store i32 %add6, i32 addrspace(1)* %dst, align 4
2835  ret void
2836}
2837