1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX8 %s
4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-NODL %s
5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-DL %s
6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-DL %s
7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-DL %s
8
9; add(mul(S0.x, S1.y),
10;     add (mul (S0.y, S1.y), S3)) -> v_dot2_{I|U}32_{I|U}16(S1, S2, S3)
11
12define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1,
13; GFX7-LABEL: udot2:
14; GFX7:       ; %bb.0: ; %entry
15; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
16; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
17; GFX7-NEXT:    s_mov_b32 s8, 0xffff
18; GFX7-NEXT:    s_mov_b32 s3, 0xf000
19; GFX7-NEXT:    s_mov_b32 s2, -1
20; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
22; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
23; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
24; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
25; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
26; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
27; GFX7-NEXT:    s_and_b32 s4, s4, s8
28; GFX7-NEXT:    v_mov_b32_e32 v0, s7
29; GFX7-NEXT:    v_mov_b32_e32 v1, s6
30; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v0, v1
31; GFX7-NEXT:    s_and_b32 s5, s5, s8
32; GFX7-NEXT:    v_mov_b32_e32 v1, s4
33; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
34; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
35; GFX7-NEXT:    s_endpgm
36;
37; GFX8-LABEL: udot2:
38; GFX8:       ; %bb.0: ; %entry
39; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
40; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
41; GFX8-NEXT:    s_mov_b32 s2, 0xffff
42; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
43; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
44; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
45; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
46; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
47; GFX8-NEXT:    s_and_b32 s6, s3, s2
48; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
49; GFX8-NEXT:    s_and_b32 s2, s4, s2
50; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
51; GFX8-NEXT:    v_mov_b32_e32 v0, s5
52; GFX8-NEXT:    v_mov_b32_e32 v1, s3
53; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
54; GFX8-NEXT:    v_mov_b32_e32 v1, s6
55; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
56; GFX8-NEXT:    v_mov_b32_e32 v0, s0
57; GFX8-NEXT:    v_mov_b32_e32 v1, s1
58; GFX8-NEXT:    flat_store_dword v[0:1], v2
59; GFX8-NEXT:    s_endpgm
60;
61; GFX9-NODL-LABEL: udot2:
62; GFX9-NODL:       ; %bb.0: ; %entry
63; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
64; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
65; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
66; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
67; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
68; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
69; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
70; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
71; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
72; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
73; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
74; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
75; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
76; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
77; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
78; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
79; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
80; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
81; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
82; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
83; GFX9-NODL-NEXT:    s_endpgm
84;
85; GFX9-DL-LABEL: udot2:
86; GFX9-DL:       ; %bb.0: ; %entry
87; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
88; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
89; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
90; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
91; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
92; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
93; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
94; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
95; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
96; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
97; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
98; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s3, v2, v3
99; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
100; GFX9-DL-NEXT:    s_endpgm
101;
102; GFX10-DL-LABEL: udot2:
103; GFX10-DL:       ; %bb.0: ; %entry
104; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
105; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
106; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
107; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
108; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
109; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
110; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
111; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
112; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
113; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
114; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
115; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s3, s2, v2
116; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
117; GFX10-DL-NEXT:    s_endpgm
118                                 <2 x i16> addrspace(1)* %src2,
119                                 i32 addrspace(1)* nocapture %dst) {
120entry:
121  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
122  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
123
124  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
125  %conv = zext i16 %s1.elt1 to i32
126  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
127  %conv2 = zext i16 %s2.elt1 to i32
128  %mul1 = mul nuw i32 %conv2, %conv
129
130  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
131  %conv3 = zext i16 %s1.elt2 to i32
132  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
133  %conv4 = zext i16 %s2.elt2 to i32
134  %mul2 = mul nuw i32 %conv4, %conv3
135
136  %s3 = load i32, i32 addrspace(1)* %dst, align 4
137  %add = add i32 %mul2, %s3
138  %add6 = add i32 %add, %mul1
139  store i32 %add6, i32 addrspace(1)* %dst, align 4
140  ret void
141}
142
143; TODO: Support this pattern
144;      add(S3,
145;          add (mul (S0.y, S1.y), mul (S0.y, S1.y))) -> v_dot2_{I|U}32_{I|U}16(S1, S2, S3)
146define amdgpu_kernel void @udot2_MulMul(<2 x i16> addrspace(1)* %src1,
147; GFX7-LABEL: udot2_MulMul:
148; GFX7:       ; %bb.0: ; %entry
149; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
150; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
151; GFX7-NEXT:    s_mov_b32 s8, 0xffff
152; GFX7-NEXT:    s_mov_b32 s3, 0xf000
153; GFX7-NEXT:    s_mov_b32 s2, -1
154; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
155; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
156; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
157; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
158; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
159; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
160; GFX7-NEXT:    s_and_b32 s4, s4, s8
161; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
162; GFX7-NEXT:    s_and_b32 s5, s5, s8
163; GFX7-NEXT:    v_mov_b32_e32 v0, s4
164; GFX7-NEXT:    v_mul_u32_u24_e32 v0, s5, v0
165; GFX7-NEXT:    v_mov_b32_e32 v1, s7
166; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
167; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
168; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
169; GFX7-NEXT:    s_endpgm
170;
171; GFX8-LABEL: udot2_MulMul:
172; GFX8:       ; %bb.0: ; %entry
173; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
174; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
175; GFX8-NEXT:    s_mov_b32 s2, 0xffff
176; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
177; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
178; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
179; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
180; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
181; GFX8-NEXT:    s_and_b32 s6, s3, s2
182; GFX8-NEXT:    s_and_b32 s2, s4, s2
183; GFX8-NEXT:    v_mov_b32_e32 v0, s6
184; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
185; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
186; GFX8-NEXT:    v_mov_b32_e32 v1, s3
187; GFX8-NEXT:    v_mul_u32_u24_e32 v0, s2, v0
188; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
189; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s5, v0
190; GFX8-NEXT:    v_mov_b32_e32 v0, s0
191; GFX8-NEXT:    v_mov_b32_e32 v1, s1
192; GFX8-NEXT:    flat_store_dword v[0:1], v2
193; GFX8-NEXT:    s_endpgm
194;
195; GFX9-NODL-LABEL: udot2_MulMul:
196; GFX9-NODL:       ; %bb.0: ; %entry
197; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
198; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
199; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
200; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
201; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
202; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
203; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
204; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
205; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
206; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
207; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s6
208; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
209; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
210; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
211; GFX9-NODL-NEXT:    v_mul_u32_u24_e32 v0, s2, v0
212; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
213; GFX9-NODL-NEXT:    v_add_u32_e32 v2, s5, v0
214; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
215; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
216; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
217; GFX9-NODL-NEXT:    s_endpgm
218;
219; GFX9-DL-LABEL: udot2_MulMul:
220; GFX9-DL:       ; %bb.0: ; %entry
221; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
222; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
223; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
224; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
225; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
226; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
227; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
228; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
229; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
230; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
231; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
232; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
233; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
234; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
235; GFX9-DL-NEXT:    v_mul_u32_u24_e32 v0, s2, v0
236; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
237; GFX9-DL-NEXT:    v_add_u32_e32 v2, s5, v0
238; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
239; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
240; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
241; GFX9-DL-NEXT:    s_endpgm
242;
243; GFX10-DL-LABEL: udot2_MulMul:
244; GFX10-DL:       ; %bb.0: ; %entry
245; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
246; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
247; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
248; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
249; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
250; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
251; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
252; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
253; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
254; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
255; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
256; GFX10-DL-NEXT:    s_and_b32 s0, s3, s2
257; GFX10-DL-NEXT:    s_and_b32 s1, s4, s2
258; GFX10-DL-NEXT:    s_lshr_b32 s2, s3, 16
259; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 16
260; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v2, s1, s0
261; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v2
262; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, s5, v2
263; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
264; GFX10-DL-NEXT:    s_endpgm
265                                        <2 x i16> addrspace(1)* %src2,
266                                        i32 addrspace(1)* nocapture %dst) {
267entry:
268  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
269  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
270
271  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
272  %conv = zext i16 %s1.elt1 to i32
273  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
274  %conv2 = zext i16 %s2.elt1 to i32
275  %mul1 = mul nuw i32 %conv2, %conv
276
277  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
278  %conv3 = zext i16 %s1.elt2 to i32
279  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
280  %conv4 = zext i16 %s2.elt2 to i32
281  %mul2 = mul nuw i32 %conv4, %conv3
282  %s3 = load i32, i32 addrspace(1)* %dst, align 4
283  %add = add i32 %mul2, %mul1
284  %add6 = add i32 %add, %s3
285  store i32 %add6, i32 addrspace(1)* %dst, align 4
286  ret void
287}
288
289define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
290; GFX7-LABEL: idot2:
291; GFX7:       ; %bb.0: ; %entry
292; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
293; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
294; GFX7-NEXT:    s_mov_b32 s3, 0xf000
295; GFX7-NEXT:    s_mov_b32 s2, -1
296; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
297; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
298; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
299; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
300; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
301; GFX7-NEXT:    s_sext_i32_i16 s7, s4
302; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
303; GFX7-NEXT:    s_sext_i32_i16 s8, s5
304; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
305; GFX7-NEXT:    v_mov_b32_e32 v0, s4
306; GFX7-NEXT:    v_mov_b32_e32 v1, s6
307; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
308; GFX7-NEXT:    v_mov_b32_e32 v1, s7
309; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
310; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
311; GFX7-NEXT:    s_endpgm
312;
313; GFX8-LABEL: idot2:
314; GFX8:       ; %bb.0: ; %entry
315; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
316; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
317; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
318; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
319; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
320; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
321; GFX8-NEXT:    v_mov_b32_e32 v0, s0
322; GFX8-NEXT:    v_mov_b32_e32 v1, s1
323; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
324; GFX8-NEXT:    s_sext_i32_i16 s0, s2
325; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
326; GFX8-NEXT:    s_sext_i32_i16 s1, s3
327; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
328; GFX8-NEXT:    v_mov_b32_e32 v2, s4
329; GFX8-NEXT:    v_mov_b32_e32 v3, s2
330; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
331; GFX8-NEXT:    v_mov_b32_e32 v3, s0
332; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
333; GFX8-NEXT:    flat_store_dword v[0:1], v2
334; GFX8-NEXT:    s_endpgm
335;
336; GFX9-NODL-LABEL: idot2:
337; GFX9-NODL:       ; %bb.0: ; %entry
338; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
339; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
340; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
341; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
342; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
343; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
344; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
345; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
346; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
347; GFX9-NODL-NEXT:    s_sext_i32_i16 s0, s2
348; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
349; GFX9-NODL-NEXT:    s_sext_i32_i16 s1, s3
350; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
351; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
352; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
353; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
354; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
355; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
356; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
357; GFX9-NODL-NEXT:    s_endpgm
358;
359; GFX9-DL-LABEL: idot2:
360; GFX9-DL:       ; %bb.0: ; %entry
361; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
362; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
363; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
364; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
365; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
366; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
367; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
368; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
369; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
370; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
371; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
372; GFX9-DL-NEXT:    v_dot2_i32_i16 v2, s3, v2, v3
373; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
374; GFX9-DL-NEXT:    s_endpgm
375;
376; GFX10-DL-LABEL: idot2:
377; GFX10-DL:       ; %bb.0: ; %entry
378; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
379; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
380; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
381; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
382; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
383; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
384; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
385; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
386; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
387; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
388; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
389; GFX10-DL-NEXT:    v_dot2_i32_i16 v2, s3, s2, v2
390; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
391; GFX10-DL-NEXT:    s_endpgm
392                                 <2 x i16> addrspace(1)* %src2,
393                                 i32 addrspace(1)* nocapture %dst) {
394entry:
395  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
396  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
397
398  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
399  %conv = sext i16 %s1.elt1 to i32
400  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
401  %conv2 = sext i16 %s2.elt1 to i32
402  %mul1 = mul nuw i32 %conv2, %conv
403
404  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
405  %conv3 = sext i16 %s1.elt2 to i32
406  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
407  %conv4 = sext i16 %s2.elt2 to i32
408  %mul2 = mul nuw i32 %conv4, %conv3
409
410  %s3 = load i32, i32 addrspace(1)* %dst, align 4
411  %add = add i32 %mul2, %s3
412  %add6 = add i32 %add, %mul1
413  store i32 %add6, i32 addrspace(1)* %dst, align 4
414  ret void
415}
416
417define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
418; GFX7-LABEL: idot2_MixedTypedMul:
419; GFX7:       ; %bb.0: ; %entry
420; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
421; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
422; GFX7-NEXT:    s_mov_b32 s3, 0xf000
423; GFX7-NEXT:    s_mov_b32 s2, -1
424; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
425; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
426; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
427; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
428; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
429; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
430; GFX7-NEXT:    s_lshr_b32 s8, s5, 16
431; GFX7-NEXT:    s_sext_i32_i16 s4, s4
432; GFX7-NEXT:    v_mov_b32_e32 v0, s7
433; GFX7-NEXT:    v_mov_b32_e32 v1, s6
434; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v0, v1
435; GFX7-NEXT:    s_sext_i32_i16 s5, s5
436; GFX7-NEXT:    v_mov_b32_e32 v1, s4
437; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
438; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
439; GFX7-NEXT:    s_endpgm
440;
441; GFX8-LABEL: idot2_MixedTypedMul:
442; GFX8:       ; %bb.0: ; %entry
443; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
444; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
445; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
446; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
447; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
448; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
449; GFX8-NEXT:    v_mov_b32_e32 v0, s0
450; GFX8-NEXT:    v_mov_b32_e32 v1, s1
451; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
452; GFX8-NEXT:    s_sext_i32_i16 s0, s2
453; GFX8-NEXT:    s_lshr_b32 s2, s2, 16
454; GFX8-NEXT:    s_sext_i32_i16 s1, s3
455; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
456; GFX8-NEXT:    v_mov_b32_e32 v2, s4
457; GFX8-NEXT:    v_mov_b32_e32 v3, s2
458; GFX8-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
459; GFX8-NEXT:    v_mov_b32_e32 v3, s0
460; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
461; GFX8-NEXT:    flat_store_dword v[0:1], v2
462; GFX8-NEXT:    s_endpgm
463;
464; GFX9-NODL-LABEL: idot2_MixedTypedMul:
465; GFX9-NODL:       ; %bb.0: ; %entry
466; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
467; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
468; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
469; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
470; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
471; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
472; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
473; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
474; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
475; GFX9-NODL-NEXT:    s_sext_i32_i16 s0, s2
476; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 16
477; GFX9-NODL-NEXT:    s_sext_i32_i16 s1, s3
478; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
479; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
480; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
481; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
482; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
483; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
484; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
485; GFX9-NODL-NEXT:    s_endpgm
486;
487; GFX9-DL-LABEL: idot2_MixedTypedMul:
488; GFX9-DL:       ; %bb.0: ; %entry
489; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
490; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
491; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
492; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
493; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
494; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
495; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
496; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
497; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
498; GFX9-DL-NEXT:    s_sext_i32_i16 s0, s2
499; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 16
500; GFX9-DL-NEXT:    s_sext_i32_i16 s1, s3
501; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
502; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
503; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
504; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s3, v3, v2
505; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s0
506; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
507; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
508; GFX9-DL-NEXT:    s_endpgm
509;
510; GFX10-DL-LABEL: idot2_MixedTypedMul:
511; GFX10-DL:       ; %bb.0: ; %entry
512; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
513; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
514; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
515; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
516; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
517; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
518; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
519; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
520; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
521; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
522; GFX10-DL-NEXT:    s_lshr_b32 s0, s2, 16
523; GFX10-DL-NEXT:    s_lshr_b32 s1, s3, 16
524; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
525; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s2
526; GFX10-DL-NEXT:    s_sext_i32_i16 s3, s3
527; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
528; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v2
529; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
530; GFX10-DL-NEXT:    s_endpgm
531                                               <2 x i16> addrspace(1)* %src2,
532                                               i32 addrspace(1)* nocapture %dst) {
533entry:
534  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
535  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
536
537  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
538  %conv = sext i16 %s1.elt1 to i32
539  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
540  %conv2 = sext i16 %s2.elt1 to i32
541  %mul1 = mul nuw i32 %conv2, %conv
542
543  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
544  %conv3 = zext i16 %s1.elt2 to i32
545  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
546  %conv4 = zext i16 %s2.elt2 to i32
547  %mul2 = mul nuw i32 %conv4, %conv3
548
549  %s3 = load i32, i32 addrspace(1)* %dst, align 4
550  %add = add i32 %mul2, %s3
551  %add6 = add i32 %add, %mul1
552  store i32 %add6, i32 addrspace(1)* %dst, align 4
553  ret void
554}
555
556define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1,
557; GFX7-LABEL: udot2_alt_AddOperands:
558; GFX7:       ; %bb.0: ; %entry
559; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
560; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
561; GFX7-NEXT:    s_mov_b32 s8, 0xffff
562; GFX7-NEXT:    s_mov_b32 s3, 0xf000
563; GFX7-NEXT:    s_mov_b32 s2, -1
564; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
565; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
566; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
567; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
568; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
569; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
570; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
571; GFX7-NEXT:    s_and_b32 s4, s4, s8
572; GFX7-NEXT:    v_mov_b32_e32 v0, s7
573; GFX7-NEXT:    v_mov_b32_e32 v1, s6
574; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v0, v1
575; GFX7-NEXT:    s_and_b32 s5, s5, s8
576; GFX7-NEXT:    v_mov_b32_e32 v1, s4
577; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
578; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
579; GFX7-NEXT:    s_endpgm
580;
581; GFX8-LABEL: udot2_alt_AddOperands:
582; GFX8:       ; %bb.0: ; %entry
583; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
584; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
585; GFX8-NEXT:    s_mov_b32 s2, 0xffff
586; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
587; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
588; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
589; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
590; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
591; GFX8-NEXT:    s_and_b32 s6, s3, s2
592; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
593; GFX8-NEXT:    s_and_b32 s2, s4, s2
594; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
595; GFX8-NEXT:    v_mov_b32_e32 v0, s5
596; GFX8-NEXT:    v_mov_b32_e32 v1, s3
597; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
598; GFX8-NEXT:    v_mov_b32_e32 v1, s6
599; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
600; GFX8-NEXT:    v_mov_b32_e32 v0, s0
601; GFX8-NEXT:    v_mov_b32_e32 v1, s1
602; GFX8-NEXT:    flat_store_dword v[0:1], v2
603; GFX8-NEXT:    s_endpgm
604;
605; GFX9-NODL-LABEL: udot2_alt_AddOperands:
606; GFX9-NODL:       ; %bb.0: ; %entry
607; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
608; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
609; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
610; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
611; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
612; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
613; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
614; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
615; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
616; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
617; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
618; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
619; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
620; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
621; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
622; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
623; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
624; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
625; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
626; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
627; GFX9-NODL-NEXT:    s_endpgm
628;
629; GFX9-DL-LABEL: udot2_alt_AddOperands:
630; GFX9-DL:       ; %bb.0: ; %entry
631; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
632; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
633; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
634; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
635; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
636; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
637; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
638; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
639; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
640; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
641; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
642; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s3, v2, v3
643; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
644; GFX9-DL-NEXT:    s_endpgm
645;
646; GFX10-DL-LABEL: udot2_alt_AddOperands:
647; GFX10-DL:       ; %bb.0: ; %entry
648; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
649; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
650; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
651; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
652; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
653; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
654; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
655; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
656; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
657; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
658; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
659; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s3, s2, v2
660; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
661; GFX10-DL-NEXT:    s_endpgm
662                                                 <2 x i16> addrspace(1)* %src2,
663                                                 i32 addrspace(1)* nocapture %dst) {
664entry:
665  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
666  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
667
668  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
669  %conv = zext i16 %s1.elt1 to i32
670  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
671  %conv2 = zext i16 %s2.elt1 to i32
672  %mul1 = mul nuw i32 %conv2, %conv
673
674  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
675  %conv3 = zext i16 %s1.elt2 to i32
676  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
677  %conv4 = zext i16 %s2.elt2 to i32
678  %mul2 = mul nuw i32 %conv4, %conv3
679
680  %s3 = load i32, i32 addrspace(1)* %dst, align 4
681  %add = add i32 %s3, %mul2
682  %add6 = add i32 %mul1, %add
683  store i32 %add6, i32 addrspace(1)* %dst, align 4
684  ret void
685}
686
687define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
688; GFX7-LABEL: idot2_MixedExt:
689; GFX7:       ; %bb.0: ; %entry
690; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
691; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
692; GFX7-NEXT:    s_mov_b32 s3, 0xf000
693; GFX7-NEXT:    s_mov_b32 s2, -1
694; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
695; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
696; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
697; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
698; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
699; GFX7-NEXT:    s_sext_i32_i16 s7, s4
700; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
701; GFX7-NEXT:    s_and_b32 s8, s5, 0xffff
702; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
703; GFX7-NEXT:    v_mov_b32_e32 v0, s4
704; GFX7-NEXT:    v_mov_b32_e32 v1, s6
705; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
706; GFX7-NEXT:    v_mov_b32_e32 v1, s7
707; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
708; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
709; GFX7-NEXT:    s_endpgm
710;
711; GFX8-LABEL: idot2_MixedExt:
712; GFX8:       ; %bb.0: ; %entry
713; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
714; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
715; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
716; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
717; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
718; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
719; GFX8-NEXT:    v_mov_b32_e32 v0, s0
720; GFX8-NEXT:    v_mov_b32_e32 v1, s1
721; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
722; GFX8-NEXT:    s_sext_i32_i16 s0, s2
723; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
724; GFX8-NEXT:    s_and_b32 s1, s3, 0xffff
725; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
726; GFX8-NEXT:    v_mov_b32_e32 v2, s4
727; GFX8-NEXT:    v_mov_b32_e32 v3, s2
728; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
729; GFX8-NEXT:    v_mov_b32_e32 v3, s0
730; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
731; GFX8-NEXT:    flat_store_dword v[0:1], v2
732; GFX8-NEXT:    s_endpgm
733;
734; GFX9-NODL-LABEL: idot2_MixedExt:
735; GFX9-NODL:       ; %bb.0: ; %entry
736; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
737; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
738; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
739; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
740; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
741; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
742; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
743; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
744; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
745; GFX9-NODL-NEXT:    s_sext_i32_i16 s0, s2
746; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
747; GFX9-NODL-NEXT:    s_and_b32 s1, s3, 0xffff
748; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
749; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
750; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
751; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
752; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
753; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
754; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
755; GFX9-NODL-NEXT:    s_endpgm
756;
757; GFX9-DL-LABEL: idot2_MixedExt:
758; GFX9-DL:       ; %bb.0: ; %entry
759; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
760; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
761; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
762; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
763; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
764; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
765; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
766; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
767; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
768; GFX9-DL-NEXT:    s_sext_i32_i16 s0, s2
769; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
770; GFX9-DL-NEXT:    s_and_b32 s1, s3, 0xffff
771; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
772; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
773; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
774; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
775; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s0
776; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
777; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
778; GFX9-DL-NEXT:    s_endpgm
779;
780; GFX10-DL-LABEL: idot2_MixedExt:
781; GFX10-DL:       ; %bb.0: ; %entry
782; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
783; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
784; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
785; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
786; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
787; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
788; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
789; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
790; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
791; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
792; GFX10-DL-NEXT:    s_ashr_i32 s0, s2, 16
793; GFX10-DL-NEXT:    s_ashr_i32 s1, s3, 16
794; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
795; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s2
796; GFX10-DL-NEXT:    s_and_b32 s3, s3, 0xffff
797; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v2
798; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v2
799; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
800; GFX10-DL-NEXT:    s_endpgm
801                                          <2 x i16> addrspace(1)* %src2,
802                                          i32 addrspace(1)* nocapture %dst) {
803entry:
804  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
805  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
806
807  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
808  %conv = sext i16 %s1.elt1 to i32
809  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
810  %conv2 = zext i16 %s2.elt1 to i32
811  %mul1 = mul nuw i32 %conv2, %conv
812
813  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
814  %conv3 = sext i16 %s1.elt2 to i32
815  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
816  %conv4 = sext i16 %s2.elt2 to i32
817  %mul2 = mul nuw i32 %conv4, %conv3
818
819  %s3 = load i32, i32 addrspace(1)* %dst, align 4
820  %add = add i32 %mul2, %s3
821  %add6 = add i32 %add, %mul1
822  store i32 %add6, i32 addrspace(1)* %dst, align 4
823  ret void
824}
825
826define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
827; GFX7-LABEL: notudot2_SameVec:
828; GFX7:       ; %bb.0: ; %entry
829; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
830; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
831; GFX7-NEXT:    s_mov_b32 s3, 0xf000
832; GFX7-NEXT:    s_mov_b32 s2, -1
833; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
834; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
835; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
836; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
837; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
838; GFX7-NEXT:    s_and_b32 s4, s4, 0xffff
839; GFX7-NEXT:    s_lshr_b32 s5, s5, 16
840; GFX7-NEXT:    v_mov_b32_e32 v0, s6
841; GFX7-NEXT:    v_mad_u32_u24 v0, s5, s5, v0
842; GFX7-NEXT:    v_mad_u32_u24 v0, s4, s4, v0
843; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
844; GFX7-NEXT:    s_endpgm
845;
846; GFX8-LABEL: notudot2_SameVec:
847; GFX8:       ; %bb.0: ; %entry
848; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
849; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
850; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
851; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
852; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
853; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
854; GFX8-NEXT:    v_mov_b32_e32 v0, s0
855; GFX8-NEXT:    v_mov_b32_e32 v1, s1
856; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
857; GFX8-NEXT:    s_and_b32 s0, s2, 0xffff
858; GFX8-NEXT:    s_lshr_b32 s1, s3, 16
859; GFX8-NEXT:    v_mov_b32_e32 v2, s4
860; GFX8-NEXT:    v_mad_u32_u24 v2, s1, s1, v2
861; GFX8-NEXT:    v_mad_u32_u24 v2, s0, s0, v2
862; GFX8-NEXT:    flat_store_dword v[0:1], v2
863; GFX8-NEXT:    s_endpgm
864;
865; GFX9-NODL-LABEL: notudot2_SameVec:
866; GFX9-NODL:       ; %bb.0: ; %entry
867; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
868; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
869; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
870; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
871; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
872; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
873; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
874; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
875; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
876; GFX9-NODL-NEXT:    s_and_b32 s0, s2, 0xffff
877; GFX9-NODL-NEXT:    s_lshr_b32 s1, s3, 16
878; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
879; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s1, s1, v2
880; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, s0, v2
881; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
882; GFX9-NODL-NEXT:    s_endpgm
883;
884; GFX9-DL-LABEL: notudot2_SameVec:
885; GFX9-DL:       ; %bb.0: ; %entry
886; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
887; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
888; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
889; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
890; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
891; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
892; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
893; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
894; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
895; GFX9-DL-NEXT:    s_and_b32 s0, s2, 0xffff
896; GFX9-DL-NEXT:    s_lshr_b32 s1, s3, 16
897; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
898; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, s1, v2
899; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s0, s0, v2
900; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
901; GFX9-DL-NEXT:    s_endpgm
902;
903; GFX10-DL-LABEL: notudot2_SameVec:
904; GFX10-DL:       ; %bb.0: ; %entry
905; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
906; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
907; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
908; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
909; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
910; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
911; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
912; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
913; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
914; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
915; GFX10-DL-NEXT:    s_and_b32 s0, s2, 0xffff
916; GFX10-DL-NEXT:    s_lshr_b32 s1, s3, 16
917; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s1, s4
918; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s0, s0, v2
919; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
920; GFX10-DL-NEXT:    s_endpgm
921                                            <2 x i16> addrspace(1)* %src2,
922                                            i32 addrspace(1)* nocapture %dst) {
923entry:
924  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
925  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
926
927  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
928  %conv = zext i16 %s1.elt1 to i32
929  %s2.elt1 = extractelement <2 x i16> %vec1, i64 0
930  %conv2 = zext i16 %s2.elt1 to i32
931  %mul1 = mul i32 %conv2, %conv
932
933  %s1.elt2 = extractelement <2 x i16> %vec2, i64 1
934  %conv3 = zext i16 %s1.elt2 to i32
935  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
936  %conv4 = zext i16 %s2.elt2 to i32
937  %mul2 = mul i32 %conv4, %conv3
938
939  %s3 = load i32, i32 addrspace(1)* %dst, align 4
940  %add = add i32 %mul2, %s3
941  %add6 = add i32 %add, %mul1
942  store i32 %add6, i32 addrspace(1)* %dst, align 4
943  ret void
944}
945
946define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1,
947; GFX7-LABEL: udot2_v4i16:
948; GFX7:       ; %bb.0: ; %entry
949; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
950; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
951; GFX7-NEXT:    s_mov_b32 s8, 0xffff
952; GFX7-NEXT:    s_mov_b32 s3, 0xf000
953; GFX7-NEXT:    s_mov_b32 s2, -1
954; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
955; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
956; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
957; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
958; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
959; GFX7-NEXT:    s_and_b32 s7, s4, s8
960; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
961; GFX7-NEXT:    s_and_b32 s8, s5, s8
962; GFX7-NEXT:    s_lshr_b32 s5, s5, 16
963; GFX7-NEXT:    v_mov_b32_e32 v0, s4
964; GFX7-NEXT:    v_mov_b32_e32 v1, s6
965; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
966; GFX7-NEXT:    v_mov_b32_e32 v1, s7
967; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
968; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
969; GFX7-NEXT:    s_endpgm
970;
971; GFX8-LABEL: udot2_v4i16:
972; GFX8:       ; %bb.0: ; %entry
973; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
974; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
975; GFX8-NEXT:    s_mov_b32 s2, 0xffff
976; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
977; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
978; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
979; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
980; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
981; GFX8-NEXT:    s_and_b32 s6, s3, s2
982; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
983; GFX8-NEXT:    s_and_b32 s2, s4, s2
984; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
985; GFX8-NEXT:    v_mov_b32_e32 v0, s5
986; GFX8-NEXT:    v_mov_b32_e32 v1, s3
987; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
988; GFX8-NEXT:    v_mov_b32_e32 v1, s6
989; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
990; GFX8-NEXT:    v_mov_b32_e32 v0, s0
991; GFX8-NEXT:    v_mov_b32_e32 v1, s1
992; GFX8-NEXT:    flat_store_dword v[0:1], v2
993; GFX8-NEXT:    s_endpgm
994;
995; GFX9-NODL-LABEL: udot2_v4i16:
996; GFX9-NODL:       ; %bb.0: ; %entry
997; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
998; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
999; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1000; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1001; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1002; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1003; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1004; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1005; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1006; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
1007; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1008; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
1009; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
1010; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1011; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1012; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1013; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
1014; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1015; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1016; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1017; GFX9-NODL-NEXT:    s_endpgm
1018;
1019; GFX9-DL-LABEL: udot2_v4i16:
1020; GFX9-DL:       ; %bb.0: ; %entry
1021; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1022; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1023; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1024; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1025; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1026; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
1027; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1028; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1029; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1030; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
1031; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
1032; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s3, v2, v3
1033; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1034; GFX9-DL-NEXT:    s_endpgm
1035;
1036; GFX10-DL-LABEL: udot2_v4i16:
1037; GFX10-DL:       ; %bb.0: ; %entry
1038; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1039; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1040; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1041; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1042; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1043; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1044; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
1045; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1046; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1047; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1048; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
1049; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s3, s2, v2
1050; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1051; GFX10-DL-NEXT:    s_endpgm
1052                                       <4 x i16> addrspace(1)* %src2,
1053                                       i32 addrspace(1)* nocapture %dst) {
1054entry:
1055  %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
1056  %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
1057
1058  %s1.elt1 = extractelement <4 x i16> %vec1, i64 0
1059  %conv = zext i16 %s1.elt1 to i32
1060  %s2.elt1 = extractelement <4 x i16> %vec2, i64 0
1061  %conv2 = zext i16 %s2.elt1 to i32
1062  %mul1 = mul i32 %conv2, %conv
1063
1064  %s1.elt2 = extractelement <4 x i16> %vec1, i64 1
1065  %conv3 = zext i16 %s1.elt2 to i32
1066  %s2.elt2 = extractelement <4 x i16> %vec2, i64 1
1067  %conv4 = zext i16 %s2.elt2 to i32
1068  %mul2 = mul i32 %conv4, %conv3
1069
1070  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1071  %add = add i32 %mul2, %s3
1072  %add6 = add i32 %add, %mul1
1073  store i32 %add6, i32 addrspace(1)* %dst, align 4
1074  ret void
1075}
1076
1077define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1,
1078; GFX7-LABEL: udot2_v4i16_Hi:
1079; GFX7:       ; %bb.0: ; %entry
1080; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1081; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1082; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1083; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1084; GFX7-NEXT:    s_mov_b32 s2, -1
1085; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1086; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x1
1087; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x1
1088; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
1089; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1090; GFX7-NEXT:    s_and_b32 s7, s4, s8
1091; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
1092; GFX7-NEXT:    s_and_b32 s8, s5, s8
1093; GFX7-NEXT:    s_lshr_b32 s5, s5, 16
1094; GFX7-NEXT:    v_mov_b32_e32 v0, s4
1095; GFX7-NEXT:    v_mov_b32_e32 v1, s6
1096; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1097; GFX7-NEXT:    v_mov_b32_e32 v1, s7
1098; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
1099; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1100; GFX7-NEXT:    s_endpgm
1101;
1102; GFX8-LABEL: udot2_v4i16_Hi:
1103; GFX8:       ; %bb.0: ; %entry
1104; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1105; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1106; GFX8-NEXT:    s_mov_b32 s2, 0xffff
1107; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1108; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x4
1109; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x4
1110; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
1111; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1112; GFX8-NEXT:    s_and_b32 s6, s3, s2
1113; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
1114; GFX8-NEXT:    s_and_b32 s2, s4, s2
1115; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
1116; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1117; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1118; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1119; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1120; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
1121; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1122; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1123; GFX8-NEXT:    flat_store_dword v[0:1], v2
1124; GFX8-NEXT:    s_endpgm
1125;
1126; GFX9-NODL-LABEL: udot2_v4i16_Hi:
1127; GFX9-NODL:       ; %bb.0: ; %entry
1128; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1129; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1130; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1131; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1132; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x4
1133; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x4
1134; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1135; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1136; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1137; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
1138; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1139; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
1140; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
1141; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1142; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1143; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1144; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
1145; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1146; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1147; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1148; GFX9-NODL-NEXT:    s_endpgm
1149;
1150; GFX9-DL-LABEL: udot2_v4i16_Hi:
1151; GFX9-DL:       ; %bb.0: ; %entry
1152; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1153; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1154; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1155; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x4
1156; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x4
1157; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
1158; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1159; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1160; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1161; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
1162; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
1163; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s3, v2, v3
1164; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1165; GFX9-DL-NEXT:    s_endpgm
1166;
1167; GFX10-DL-LABEL: udot2_v4i16_Hi:
1168; GFX10-DL:       ; %bb.0: ; %entry
1169; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1170; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1171; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1172; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1173; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x4
1174; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x4
1175; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
1176; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1177; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1178; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1179; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
1180; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s3, s2, v2
1181; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1182; GFX10-DL-NEXT:    s_endpgm
1183                                          <4 x i16> addrspace(1)* %src2,
1184                                          i32 addrspace(1)* nocapture %dst) {
1185entry:
1186  %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
1187  %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
1188
1189  %s1.elt1 = extractelement <4 x i16> %vec1, i64 2
1190  %conv = zext i16 %s1.elt1 to i32
1191  %s2.elt1 = extractelement <4 x i16> %vec2, i64 2
1192  %conv2 = zext i16 %s2.elt1 to i32
1193  %mul1 = mul i32 %conv2, %conv
1194
1195  %s1.elt2 = extractelement <4 x i16> %vec1, i64 3
1196  %conv3 = zext i16 %s1.elt2 to i32
1197  %s2.elt2 = extractelement <4 x i16> %vec2, i64 3
1198  %conv4 = zext i16 %s2.elt2 to i32
1199  %mul2 = mul i32 %conv4, %conv3
1200
1201  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1202  %add = add i32 %mul2, %s3
1203  %add6 = add i32 %add, %mul1
1204  store i32 %add6, i32 addrspace(1)* %dst, align 4
1205  ret void
1206}
1207
1208define amdgpu_kernel void @notudot2_v4i16_Even(<4 x i16> addrspace(1)* %src1,
1209; GFX7-LABEL: notudot2_v4i16_Even:
1210; GFX7:       ; %bb.0: ; %entry
1211; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1212; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1213; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1214; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1215; GFX7-NEXT:    s_mov_b32 s2, -1
1216; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1217; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
1218; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
1219; GFX7-NEXT:    s_load_dword s9, s[0:1], 0x0
1220; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1221; GFX7-NEXT:    s_and_b32 s5, s5, s8
1222; GFX7-NEXT:    s_and_b32 s4, s4, s8
1223; GFX7-NEXT:    s_and_b32 s7, s7, s8
1224; GFX7-NEXT:    v_mov_b32_e32 v0, s5
1225; GFX7-NEXT:    v_mov_b32_e32 v1, s9
1226; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
1227; GFX7-NEXT:    s_and_b32 s6, s6, s8
1228; GFX7-NEXT:    v_mov_b32_e32 v1, s4
1229; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
1230; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1231; GFX7-NEXT:    s_endpgm
1232;
1233; GFX8-LABEL: notudot2_v4i16_Even:
1234; GFX8:       ; %bb.0: ; %entry
1235; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1236; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1237; GFX8-NEXT:    s_mov_b32 s8, 0xffff
1238; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1239; GFX8-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1240; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1241; GFX8-NEXT:    s_load_dword s6, s[0:1], 0x0
1242; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1243; GFX8-NEXT:    s_and_b32 s3, s3, s8
1244; GFX8-NEXT:    s_and_b32 s2, s2, s8
1245; GFX8-NEXT:    s_and_b32 s5, s5, s8
1246; GFX8-NEXT:    v_mov_b32_e32 v0, s6
1247; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1248; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1249; GFX8-NEXT:    s_and_b32 s4, s4, s8
1250; GFX8-NEXT:    v_mov_b32_e32 v1, s2
1251; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1252; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1253; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1254; GFX8-NEXT:    flat_store_dword v[0:1], v2
1255; GFX8-NEXT:    s_endpgm
1256;
1257; GFX9-NODL-LABEL: notudot2_v4i16_Even:
1258; GFX9-NODL:       ; %bb.0: ; %entry
1259; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1260; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1261; GFX9-NODL-NEXT:    s_mov_b32 s8, 0xffff
1262; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1263; GFX9-NODL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1264; GFX9-NODL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1265; GFX9-NODL-NEXT:    s_load_dword s6, s[0:1], 0x0
1266; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1267; GFX9-NODL-NEXT:    s_and_b32 s3, s3, s8
1268; GFX9-NODL-NEXT:    s_and_b32 s2, s2, s8
1269; GFX9-NODL-NEXT:    s_and_b32 s5, s5, s8
1270; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s6
1271; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1272; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1273; GFX9-NODL-NEXT:    s_and_b32 s4, s4, s8
1274; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
1275; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1276; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1277; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1278; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1279; GFX9-NODL-NEXT:    s_endpgm
1280;
1281; GFX9-DL-LABEL: notudot2_v4i16_Even:
1282; GFX9-DL:       ; %bb.0: ; %entry
1283; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1284; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1285; GFX9-DL-NEXT:    s_mov_b32 s8, 0xffff
1286; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1287; GFX9-DL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1288; GFX9-DL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1289; GFX9-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
1290; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1291; GFX9-DL-NEXT:    s_and_b32 s3, s3, s8
1292; GFX9-DL-NEXT:    s_and_b32 s2, s2, s8
1293; GFX9-DL-NEXT:    s_and_b32 s5, s5, s8
1294; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
1295; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1296; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1297; GFX9-DL-NEXT:    s_and_b32 s4, s4, s8
1298; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
1299; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1300; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1301; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1302; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1303; GFX9-DL-NEXT:    s_endpgm
1304;
1305; GFX10-DL-LABEL: notudot2_v4i16_Even:
1306; GFX10-DL:       ; %bb.0: ; %entry
1307; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1308; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1309; GFX10-DL-NEXT:    s_mov_b32 s8, 0xffff
1310; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1311; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1312; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1313; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1314; GFX10-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
1315; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1316; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1317; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1318; GFX10-DL-NEXT:    s_and_b32 s0, s3, s8
1319; GFX10-DL-NEXT:    s_and_b32 s1, s5, s8
1320; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s6
1321; GFX10-DL-NEXT:    s_and_b32 s2, s2, s8
1322; GFX10-DL-NEXT:    s_and_b32 s3, s4, s8
1323; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
1324; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v2
1325; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1326; GFX10-DL-NEXT:    s_endpgm
1327                                               <4 x i16> addrspace(1)* %src2,
1328                                               i32 addrspace(1)* nocapture %dst) {
1329entry:
1330  %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
1331  %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
1332
1333  %s1.elt1 = extractelement <4 x i16> %vec1, i64 0
1334  %conv = zext i16 %s1.elt1 to i32
1335  %s2.elt1 = extractelement <4 x i16> %vec2, i64 0
1336  %conv2 = zext i16 %s2.elt1 to i32
1337  %mul1 = mul i32 %conv2, %conv
1338
1339  %s1.elt2 = extractelement <4 x i16> %vec1, i64 2
1340  %conv3 = zext i16 %s1.elt2 to i32
1341  %s2.elt2 = extractelement <4 x i16> %vec2, i64 2
1342  %conv4 = zext i16 %s2.elt2 to i32
1343  %mul2 = mul i32 %conv4, %conv3
1344
1345  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1346  %add = add i32 %mul2, %s3
1347  %add6 = add i32 %add, %mul1
1348  store i32 %add6, i32 addrspace(1)* %dst, align 4
1349  ret void
1350}
1351
1352define amdgpu_kernel void @notudot2_v4i16_Middle(<4 x i16> addrspace(1)* %src1,
1353; GFX7-LABEL: notudot2_v4i16_Middle:
1354; GFX7:       ; %bb.0: ; %entry
1355; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1356; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1357; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1358; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1359; GFX7-NEXT:    s_mov_b32 s2, -1
1360; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1361; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
1362; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
1363; GFX7-NEXT:    s_load_dword s9, s[0:1], 0x0
1364; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1365; GFX7-NEXT:    s_and_b32 s5, s5, s8
1366; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
1367; GFX7-NEXT:    s_and_b32 s7, s7, s8
1368; GFX7-NEXT:    v_mov_b32_e32 v0, s5
1369; GFX7-NEXT:    v_mov_b32_e32 v1, s9
1370; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
1371; GFX7-NEXT:    s_lshr_b32 s6, s6, 16
1372; GFX7-NEXT:    v_mov_b32_e32 v1, s4
1373; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
1374; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1375; GFX7-NEXT:    s_endpgm
1376;
1377; GFX8-LABEL: notudot2_v4i16_Middle:
1378; GFX8:       ; %bb.0: ; %entry
1379; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1380; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1381; GFX8-NEXT:    s_mov_b32 s8, 0xffff
1382; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1383; GFX8-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1384; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1385; GFX8-NEXT:    s_load_dword s6, s[0:1], 0x0
1386; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1387; GFX8-NEXT:    s_and_b32 s3, s3, s8
1388; GFX8-NEXT:    s_lshr_b32 s2, s2, 16
1389; GFX8-NEXT:    s_and_b32 s5, s5, s8
1390; GFX8-NEXT:    v_mov_b32_e32 v0, s6
1391; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1392; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1393; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
1394; GFX8-NEXT:    v_mov_b32_e32 v1, s2
1395; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1396; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1397; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1398; GFX8-NEXT:    flat_store_dword v[0:1], v2
1399; GFX8-NEXT:    s_endpgm
1400;
1401; GFX9-NODL-LABEL: notudot2_v4i16_Middle:
1402; GFX9-NODL:       ; %bb.0: ; %entry
1403; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1404; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1405; GFX9-NODL-NEXT:    s_mov_b32 s8, 0xffff
1406; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1407; GFX9-NODL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1408; GFX9-NODL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1409; GFX9-NODL-NEXT:    s_load_dword s6, s[0:1], 0x0
1410; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1411; GFX9-NODL-NEXT:    s_and_b32 s3, s3, s8
1412; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 16
1413; GFX9-NODL-NEXT:    s_and_b32 s5, s5, s8
1414; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s6
1415; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1416; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1417; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
1418; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
1419; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1420; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1421; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1422; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1423; GFX9-NODL-NEXT:    s_endpgm
1424;
1425; GFX9-DL-LABEL: notudot2_v4i16_Middle:
1426; GFX9-DL:       ; %bb.0: ; %entry
1427; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1428; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1429; GFX9-DL-NEXT:    s_mov_b32 s8, 0xffff
1430; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1431; GFX9-DL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1432; GFX9-DL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1433; GFX9-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
1434; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1435; GFX9-DL-NEXT:    s_and_b32 s3, s3, s8
1436; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 16
1437; GFX9-DL-NEXT:    s_and_b32 s5, s5, s8
1438; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
1439; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1440; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1441; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
1442; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
1443; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1444; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1445; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1446; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1447; GFX9-DL-NEXT:    s_endpgm
1448;
1449; GFX10-DL-LABEL: notudot2_v4i16_Middle:
1450; GFX10-DL:       ; %bb.0: ; %entry
1451; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1452; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1453; GFX10-DL-NEXT:    s_mov_b32 s8, 0xffff
1454; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1455; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1456; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1457; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1458; GFX10-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
1459; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1460; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1461; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1462; GFX10-DL-NEXT:    s_and_b32 s0, s3, s8
1463; GFX10-DL-NEXT:    s_and_b32 s1, s5, s8
1464; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s6
1465; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 16
1466; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 16
1467; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
1468; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v2
1469; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1470; GFX10-DL-NEXT:    s_endpgm
1471                                                 <4 x i16> addrspace(1)* %src2,
1472                                                 i32 addrspace(1)* nocapture %dst) {
1473entry:
1474  %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
1475  %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
1476
1477  %s1.elt1 = extractelement <4 x i16> %vec1, i64 1
1478  %conv = zext i16 %s1.elt1 to i32
1479  %s2.elt1 = extractelement <4 x i16> %vec2, i64 1
1480  %conv2 = zext i16 %s2.elt1 to i32
1481  %mul1 = mul i32 %conv2, %conv
1482
1483  %s1.elt2 = extractelement <4 x i16> %vec1, i64 2
1484  %conv3 = zext i16 %s1.elt2 to i32
1485  %s2.elt2 = extractelement <4 x i16> %vec2, i64 2
1486  %conv4 = zext i16 %s2.elt2 to i32
1487  %mul2 = mul i32 %conv4, %conv3
1488
1489  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1490  %add = add i32 %mul2, %s3
1491  %add6 = add i32 %add, %mul1
1492  store i32 %add6, i32 addrspace(1)* %dst, align 4
1493  ret void
1494}
1495
1496define amdgpu_kernel void @notudot2_DiffIndex(<2 x i16> addrspace(1)* %src1,
1497; GFX7-LABEL: notudot2_DiffIndex:
1498; GFX7:       ; %bb.0: ; %entry
1499; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1500; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1501; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1502; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1503; GFX7-NEXT:    s_mov_b32 s2, -1
1504; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1505; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1506; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1507; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
1508; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1509; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
1510; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
1511; GFX7-NEXT:    s_and_b32 s4, s4, s8
1512; GFX7-NEXT:    s_and_b32 s5, s5, s8
1513; GFX7-NEXT:    v_mov_b32_e32 v0, s7
1514; GFX7-NEXT:    v_mov_b32_e32 v1, s6
1515; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1516; GFX7-NEXT:    v_mov_b32_e32 v1, s4
1517; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v1, v0
1518; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1519; GFX7-NEXT:    s_endpgm
1520;
1521; GFX8-LABEL: notudot2_DiffIndex:
1522; GFX8:       ; %bb.0: ; %entry
1523; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1524; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1525; GFX8-NEXT:    s_mov_b32 s2, 0xffff
1526; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1527; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1528; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1529; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
1530; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1531; GFX8-NEXT:    s_and_b32 s6, s3, s2
1532; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
1533; GFX8-NEXT:    s_and_b32 s2, s4, s2
1534; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1535; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1536; GFX8-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1537; GFX8-NEXT:    s_lshr_b32 s7, s4, 16
1538; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1539; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v1, v0
1540; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1541; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1542; GFX8-NEXT:    flat_store_dword v[0:1], v2
1543; GFX8-NEXT:    s_endpgm
1544;
1545; GFX9-NODL-LABEL: notudot2_DiffIndex:
1546; GFX9-NODL:       ; %bb.0: ; %entry
1547; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1548; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1549; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1550; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1551; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1552; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1553; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1554; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1555; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1556; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
1557; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1558; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
1559; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1560; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1561; GFX9-NODL-NEXT:    s_lshr_b32 s7, s4, 16
1562; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1563; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s7, v1, v0
1564; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1565; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1566; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1567; GFX9-NODL-NEXT:    s_endpgm
1568;
1569; GFX9-DL-LABEL: notudot2_DiffIndex:
1570; GFX9-DL:       ; %bb.0: ; %entry
1571; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1572; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1573; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
1574; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1575; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1576; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1577; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1578; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1579; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
1580; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
1581; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
1582; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
1583; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1584; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1585; GFX9-DL-NEXT:    s_lshr_b32 s7, s4, 16
1586; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
1587; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v1, v0
1588; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1589; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1590; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1591; GFX9-DL-NEXT:    s_endpgm
1592;
1593; GFX10-DL-LABEL: notudot2_DiffIndex:
1594; GFX10-DL:       ; %bb.0: ; %entry
1595; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1596; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1597; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
1598; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1599; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1600; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1601; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1602; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1603; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1604; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1605; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1606; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 16
1607; GFX10-DL-NEXT:    s_and_b32 s1, s4, s2
1608; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
1609; GFX10-DL-NEXT:    s_and_b32 s2, s3, s2
1610; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 16
1611; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
1612; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v2
1613; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1614; GFX10-DL-NEXT:    s_endpgm
1615                                              <2 x i16> addrspace(1)* %src2,
1616                                              i32 addrspace(1)* nocapture %dst) {
1617entry:
1618  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
1619  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
1620
1621  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
1622  %conv = zext i16 %s1.elt1 to i32
1623  %s2.elt1 = extractelement <2 x i16> %vec2, i64 1
1624  %conv2 = zext i16 %s2.elt1 to i32
1625  %mul1 = mul i32 %conv2, %conv
1626
1627  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
1628  %conv3 = zext i16 %s1.elt2 to i32
1629  %s2.elt2 = extractelement <2 x i16> %vec2, i64 0
1630  %conv4 = zext i16 %s2.elt2 to i32
1631  %mul2 = mul i32 %conv4, %conv3
1632
1633  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1634  %add = add i32 %mul2, %s3
1635  %add6 = add i32 %add, %mul1
1636  store i32 %add6, i32 addrspace(1)* %dst, align 4
1637  ret void
1638}
1639
1640define amdgpu_kernel void @udot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1,
1641; GFX7-LABEL: udot2_MultipleUses_add1:
1642; GFX7:       ; %bb.0: ; %entry
1643; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1644; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1645; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1646; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1647; GFX7-NEXT:    s_mov_b32 s2, -1
1648; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1649; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1650; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1651; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
1652; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1653; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
1654; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
1655; GFX7-NEXT:    s_and_b32 s4, s4, s8
1656; GFX7-NEXT:    v_mov_b32_e32 v0, s7
1657; GFX7-NEXT:    v_mov_b32_e32 v1, s6
1658; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v0, v1
1659; GFX7-NEXT:    s_and_b32 s5, s5, s8
1660; GFX7-NEXT:    v_mov_b32_e32 v1, s4
1661; GFX7-NEXT:    v_mad_u32_u24 v1, s5, v1, v0
1662; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
1663; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1664; GFX7-NEXT:    s_endpgm
1665;
1666; GFX8-LABEL: udot2_MultipleUses_add1:
1667; GFX8:       ; %bb.0: ; %entry
1668; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1669; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1670; GFX8-NEXT:    s_mov_b32 s2, 0xffff
1671; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1672; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1673; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1674; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
1675; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1676; GFX8-NEXT:    s_and_b32 s6, s3, s2
1677; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
1678; GFX8-NEXT:    s_and_b32 s2, s4, s2
1679; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
1680; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1681; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1682; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1683; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1684; GFX8-NEXT:    v_mad_u32_u24 v1, s2, v1, v0
1685; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v1
1686; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1687; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1688; GFX8-NEXT:    flat_store_dword v[0:1], v2
1689; GFX8-NEXT:    s_endpgm
1690;
1691; GFX9-NODL-LABEL: udot2_MultipleUses_add1:
1692; GFX9-NODL:       ; %bb.0: ; %entry
1693; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1694; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1695; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1696; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1697; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1698; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1699; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1700; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1701; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1702; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
1703; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1704; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
1705; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
1706; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1707; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1708; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1709; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s2, v1, v0
1710; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v1, v0
1711; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1712; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1713; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1714; GFX9-NODL-NEXT:    s_endpgm
1715;
1716; GFX9-DL-LABEL: udot2_MultipleUses_add1:
1717; GFX9-DL:       ; %bb.0: ; %entry
1718; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1719; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1720; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
1721; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1722; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1723; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1724; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1725; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1726; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
1727; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
1728; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
1729; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
1730; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
1731; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1732; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1733; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
1734; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s2, v1, v0
1735; GFX9-DL-NEXT:    v_add_u32_e32 v2, v1, v0
1736; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1737; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1738; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1739; GFX9-DL-NEXT:    s_endpgm
1740;
1741; GFX10-DL-LABEL: udot2_MultipleUses_add1:
1742; GFX10-DL:       ; %bb.0: ; %entry
1743; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1744; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1745; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
1746; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1747; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1748; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1749; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1750; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1751; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1752; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1753; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1754; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 16
1755; GFX10-DL-NEXT:    s_lshr_b32 s1, s4, 16
1756; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
1757; GFX10-DL-NEXT:    s_and_b32 s3, s3, s2
1758; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
1759; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
1760; GFX10-DL-NEXT:    v_mad_u32_u24 v3, s2, s3, v2
1761; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
1762; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1763; GFX10-DL-NEXT:    s_endpgm
1764                                                   <2 x i16> addrspace(1)* %src2,
1765                                                   i32 addrspace(1)* nocapture %dst) {
1766entry:
1767  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
1768  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
1769
1770  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
1771  %conv = zext i16 %s1.elt1 to i32
1772  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
1773  %conv2 = zext i16 %s2.elt1 to i32
1774  %mul1 = mul i32 %conv2, %conv
1775
1776  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
1777  %conv3 = zext i16 %s1.elt2 to i32
1778  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
1779  %conv4 = zext i16 %s2.elt2 to i32
1780  %mul2 = mul i32 %conv4, %conv3
1781
1782  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1783  %add1 = add i32 %mul2, %s3
1784  %add2 = add i32 %add1, %mul1
1785
1786  %res = add i32 %add2, %add1
1787  store i32 %res, i32 addrspace(1)* %dst, align 4
1788  ret void
1789}
1790
1791define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1,
1792; GFX7-LABEL: idot2_MultipleUses_add1:
1793; GFX7:       ; %bb.0: ; %entry
1794; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1795; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1796; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1797; GFX7-NEXT:    s_mov_b32 s2, -1
1798; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1799; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1800; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1801; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
1802; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1803; GFX7-NEXT:    s_sext_i32_i16 s7, s4
1804; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
1805; GFX7-NEXT:    s_sext_i32_i16 s8, s5
1806; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
1807; GFX7-NEXT:    v_mov_b32_e32 v0, s4
1808; GFX7-NEXT:    v_mov_b32_e32 v1, s6
1809; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
1810; GFX7-NEXT:    v_mov_b32_e32 v1, s7
1811; GFX7-NEXT:    v_mad_i32_i24 v1, s8, v1, v0
1812; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
1813; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1814; GFX7-NEXT:    s_endpgm
1815;
1816; GFX8-LABEL: idot2_MultipleUses_add1:
1817; GFX8:       ; %bb.0: ; %entry
1818; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1819; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1820; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1821; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
1822; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
1823; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
1824; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1825; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1826; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1827; GFX8-NEXT:    s_sext_i32_i16 s0, s2
1828; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
1829; GFX8-NEXT:    s_sext_i32_i16 s1, s3
1830; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
1831; GFX8-NEXT:    v_mov_b32_e32 v2, s4
1832; GFX8-NEXT:    v_mov_b32_e32 v3, s2
1833; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
1834; GFX8-NEXT:    v_mov_b32_e32 v3, s0
1835; GFX8-NEXT:    v_mad_i32_i24 v3, s1, v3, v2
1836; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
1837; GFX8-NEXT:    flat_store_dword v[0:1], v2
1838; GFX8-NEXT:    s_endpgm
1839;
1840; GFX9-NODL-LABEL: idot2_MultipleUses_add1:
1841; GFX9-NODL:       ; %bb.0: ; %entry
1842; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1843; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1844; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1845; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
1846; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
1847; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
1848; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1849; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1850; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1851; GFX9-NODL-NEXT:    s_sext_i32_i16 s0, s2
1852; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
1853; GFX9-NODL-NEXT:    s_sext_i32_i16 s1, s3
1854; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
1855; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
1856; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
1857; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
1858; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
1859; GFX9-NODL-NEXT:    v_mad_i32_i24 v3, s1, v3, v2
1860; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v3, v2
1861; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1862; GFX9-NODL-NEXT:    s_endpgm
1863;
1864; GFX9-DL-LABEL: idot2_MultipleUses_add1:
1865; GFX9-DL:       ; %bb.0: ; %entry
1866; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1867; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1868; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1869; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1870; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1871; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
1872; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1873; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1874; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1875; GFX9-DL-NEXT:    s_sext_i32_i16 s0, s2
1876; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
1877; GFX9-DL-NEXT:    s_sext_i32_i16 s1, s3
1878; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
1879; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
1880; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
1881; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
1882; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s0
1883; GFX9-DL-NEXT:    v_mad_i32_i24 v3, s1, v3, v2
1884; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
1885; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1886; GFX9-DL-NEXT:    s_endpgm
1887;
1888; GFX10-DL-LABEL: idot2_MultipleUses_add1:
1889; GFX10-DL:       ; %bb.0: ; %entry
1890; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1891; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1892; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1893; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1894; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1895; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1896; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
1897; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
1898; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
1899; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1900; GFX10-DL-NEXT:    s_ashr_i32 s0, s2, 16
1901; GFX10-DL-NEXT:    s_ashr_i32 s1, s3, 16
1902; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
1903; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s2
1904; GFX10-DL-NEXT:    s_sext_i32_i16 s3, s3
1905; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v2
1906; GFX10-DL-NEXT:    v_mad_i32_i24 v3, s3, s2, v2
1907; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v3, v2
1908; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1909; GFX10-DL-NEXT:    s_endpgm
1910                                                   <2 x i16> addrspace(1)* %src2,
1911                                                   i32 addrspace(1)* nocapture %dst) {
1912entry:
1913  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
1914  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
1915
1916  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
1917  %conv = sext i16 %s1.elt1 to i32
1918  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
1919  %conv2 = sext i16 %s2.elt1 to i32
1920  %mul1 = mul i32 %conv2, %conv
1921
1922  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
1923  %conv3 = sext i16 %s1.elt2 to i32
1924  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
1925  %conv4 = sext i16 %s2.elt2 to i32
1926  %mul2 = mul i32 %conv4, %conv3
1927
1928  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1929  %add1 = add i32 %mul2, %s3
1930  %add2 = add i32 %add1, %mul1
1931
1932  %res = add i32 %add2, %add1
1933  store i32 %res, i32 addrspace(1)* %dst, align 4
1934  ret void
1935}
1936
1937define amdgpu_kernel void @udot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1,
1938; GFX7-LABEL: udot2_MultipleUses_mul1:
1939; GFX7:       ; %bb.0: ; %entry
1940; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1941; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1942; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1943; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1944; GFX7-NEXT:    s_mov_b32 s2, -1
1945; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1946; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1947; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1948; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
1949; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1950; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
1951; GFX7-NEXT:    s_and_b32 s4, s4, s8
1952; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
1953; GFX7-NEXT:    s_and_b32 s5, s5, s8
1954; GFX7-NEXT:    v_mov_b32_e32 v0, s4
1955; GFX7-NEXT:    v_mov_b32_e32 v1, s6
1956; GFX7-NEXT:    v_mad_u32_u24 v1, s5, v0, v1
1957; GFX7-NEXT:    v_mov_b32_e32 v2, s7
1958; GFX7-NEXT:    v_mad_u32_u24 v1, s9, v2, v1
1959; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1960; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1961; GFX7-NEXT:    s_endpgm
1962;
1963; GFX8-LABEL: udot2_MultipleUses_mul1:
1964; GFX8:       ; %bb.0: ; %entry
1965; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1966; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1967; GFX8-NEXT:    s_mov_b32 s2, 0xffff
1968; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1969; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1970; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1971; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
1972; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1973; GFX8-NEXT:    s_and_b32 s6, s3, s2
1974; GFX8-NEXT:    s_and_b32 s2, s4, s2
1975; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
1976; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1977; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1978; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
1979; GFX8-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1980; GFX8-NEXT:    v_mov_b32_e32 v2, s3
1981; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v2, v0
1982; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
1983; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1984; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1985; GFX8-NEXT:    flat_store_dword v[0:1], v2
1986; GFX8-NEXT:    s_endpgm
1987;
1988; GFX9-NODL-LABEL: udot2_MultipleUses_mul1:
1989; GFX9-NODL:       ; %bb.0: ; %entry
1990; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1991; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1992; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1993; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1994; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1995; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1996; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1997; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1998; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1999; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
2000; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
2001; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
2002; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
2003; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
2004; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
2005; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s3
2006; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v2, v0
2007; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2008; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2009; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2010; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2011; GFX9-NODL-NEXT:    s_endpgm
2012;
2013; GFX9-DL-LABEL: udot2_MultipleUses_mul1:
2014; GFX9-DL:       ; %bb.0: ; %entry
2015; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2016; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2017; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
2018; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2019; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
2020; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
2021; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
2022; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2023; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
2024; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
2025; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
2026; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
2027; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
2028; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
2029; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
2030; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
2031; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v2, v0
2032; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2033; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2034; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2035; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2036; GFX9-DL-NEXT:    s_endpgm
2037;
2038; GFX10-DL-LABEL: udot2_MultipleUses_mul1:
2039; GFX10-DL:       ; %bb.0: ; %entry
2040; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2041; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2042; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
2043; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2044; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2045; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
2046; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
2047; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
2048; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2049; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2050; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2051; GFX10-DL-NEXT:    s_and_b32 s0, s3, s2
2052; GFX10-DL-NEXT:    s_and_b32 s1, s4, s2
2053; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
2054; GFX10-DL-NEXT:    s_lshr_b32 s2, s3, 16
2055; GFX10-DL-NEXT:    s_lshr_b32 s3, s4, 16
2056; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
2057; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s3, s2, v2
2058; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
2059; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2060; GFX10-DL-NEXT:    s_endpgm
2061                                                   <2 x i16> addrspace(1)* %src2,
2062                                                   i32 addrspace(1)* nocapture %dst) {
2063entry:
2064  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2065  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2066
2067  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
2068  %conv = zext i16 %s1.elt1 to i32
2069  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
2070  %conv2 = zext i16 %s2.elt1 to i32
2071  %mul1 = mul i32 %conv2, %conv
2072
2073  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
2074  %conv3 = zext i16 %s1.elt2 to i32
2075  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
2076  %conv4 = zext i16 %s2.elt2 to i32
2077  %mul2 = mul i32 %conv4, %conv3
2078
2079  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2080  %add0 = add i32 %mul1, %s3
2081
2082  %add1 = add i32 %mul2, %add0
2083  %add2 = add i32 %add1, %mul1
2084
2085  store i32 %add2, i32 addrspace(1)* %dst, align 4
2086  ret void
2087}
2088
2089define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1,
2090; GFX7-LABEL: idot2_MultipleUses_mul1:
2091; GFX7:       ; %bb.0: ; %entry
2092; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2093; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2094; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2095; GFX7-NEXT:    s_mov_b32 s2, -1
2096; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2097; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2098; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2099; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
2100; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2101; GFX7-NEXT:    s_sext_i32_i16 s7, s4
2102; GFX7-NEXT:    s_sext_i32_i16 s8, s5
2103; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
2104; GFX7-NEXT:    v_mov_b32_e32 v0, s7
2105; GFX7-NEXT:    v_mov_b32_e32 v1, s6
2106; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
2107; GFX7-NEXT:    v_mad_i32_i24 v1, s8, v0, v1
2108; GFX7-NEXT:    v_mov_b32_e32 v2, s4
2109; GFX7-NEXT:    v_mad_i32_i24 v1, s5, v2, v1
2110; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v0, v1
2111; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2112; GFX7-NEXT:    s_endpgm
2113;
2114; GFX8-LABEL: idot2_MultipleUses_mul1:
2115; GFX8:       ; %bb.0: ; %entry
2116; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2117; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2118; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2119; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
2120; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
2121; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
2122; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2123; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2124; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2125; GFX8-NEXT:    s_sext_i32_i16 s0, s2
2126; GFX8-NEXT:    s_sext_i32_i16 s1, s3
2127; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
2128; GFX8-NEXT:    v_mov_b32_e32 v2, s4
2129; GFX8-NEXT:    v_mov_b32_e32 v3, s0
2130; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
2131; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
2132; GFX8-NEXT:    v_mov_b32_e32 v4, s2
2133; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v4, v2
2134; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
2135; GFX8-NEXT:    flat_store_dword v[0:1], v2
2136; GFX8-NEXT:    s_endpgm
2137;
2138; GFX9-NODL-LABEL: idot2_MultipleUses_mul1:
2139; GFX9-NODL:       ; %bb.0: ; %entry
2140; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2141; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2142; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2143; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
2144; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
2145; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
2146; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2147; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2148; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2149; GFX9-NODL-NEXT:    s_sext_i32_i16 s0, s2
2150; GFX9-NODL-NEXT:    s_sext_i32_i16 s1, s3
2151; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
2152; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
2153; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
2154; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
2155; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
2156; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s2
2157; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v4, v2
2158; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
2159; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2160; GFX9-NODL-NEXT:    s_endpgm
2161;
2162; GFX9-DL-LABEL: idot2_MultipleUses_mul1:
2163; GFX9-DL:       ; %bb.0: ; %entry
2164; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2165; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2166; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2167; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2168; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2169; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
2170; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2171; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2172; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2173; GFX9-DL-NEXT:    s_sext_i32_i16 s0, s2
2174; GFX9-DL-NEXT:    s_sext_i32_i16 s1, s3
2175; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
2176; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
2177; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s0
2178; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
2179; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
2180; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s2
2181; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s3, v4, v2
2182; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
2183; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2184; GFX9-DL-NEXT:    s_endpgm
2185;
2186; GFX10-DL-LABEL: idot2_MultipleUses_mul1:
2187; GFX10-DL:       ; %bb.0: ; %entry
2188; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2189; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2190; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2191; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2192; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2193; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2194; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
2195; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2196; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2197; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2198; GFX10-DL-NEXT:    s_sext_i32_i16 s0, s2
2199; GFX10-DL-NEXT:    s_sext_i32_i16 s1, s3
2200; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
2201; GFX10-DL-NEXT:    s_ashr_i32 s2, s2, 16
2202; GFX10-DL-NEXT:    s_ashr_i32 s3, s3, 16
2203; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v2
2204; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v2
2205; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v2
2206; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2207; GFX10-DL-NEXT:    s_endpgm
2208                                                   <2 x i16> addrspace(1)* %src2,
2209                                                   i32 addrspace(1)* nocapture %dst) {
2210entry:
2211  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2212  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2213
2214  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
2215  %conv = sext i16 %s1.elt1 to i32
2216  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
2217  %conv2 = sext i16 %s2.elt1 to i32
2218  %mul1 = mul i32 %conv2, %conv
2219
2220  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
2221  %conv3 = sext i16 %s1.elt2 to i32
2222  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
2223  %conv4 = sext i16 %s2.elt2 to i32
2224  %mul2 = mul i32 %conv4, %conv3
2225
2226  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2227  %add0 = add i32 %mul1, %s3
2228
2229  %add1 = add i32 %mul2, %add0
2230  %add2 = add i32 %add1, %mul1
2231
2232  store i32 %add2, i32 addrspace(1)* %dst, align 4
2233  ret void
2234}
2235
2236define amdgpu_kernel void @udot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1,
2237; GFX7-LABEL: udot2_MultipleUses_mul2:
2238; GFX7:       ; %bb.0: ; %entry
2239; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2240; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2241; GFX7-NEXT:    s_mov_b32 s8, 0xffff
2242; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2243; GFX7-NEXT:    s_mov_b32 s2, -1
2244; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2245; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2246; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2247; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
2248; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2249; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
2250; GFX7-NEXT:    s_lshr_b32 s9, s5, 16
2251; GFX7-NEXT:    v_mov_b32_e32 v0, s7
2252; GFX7-NEXT:    v_mov_b32_e32 v1, s6
2253; GFX7-NEXT:    v_mad_u32_u24 v1, s9, v0, v1
2254; GFX7-NEXT:    s_and_b32 s4, s4, s8
2255; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v0, v1
2256; GFX7-NEXT:    s_and_b32 s5, s5, s8
2257; GFX7-NEXT:    v_mov_b32_e32 v1, s4
2258; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
2259; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2260; GFX7-NEXT:    s_endpgm
2261;
2262; GFX8-LABEL: udot2_MultipleUses_mul2:
2263; GFX8:       ; %bb.0: ; %entry
2264; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2265; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2266; GFX8-NEXT:    s_mov_b32 s2, 0xffff
2267; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2268; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
2269; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
2270; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
2271; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2272; GFX8-NEXT:    s_and_b32 s6, s3, s2
2273; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
2274; GFX8-NEXT:    s_and_b32 s2, s4, s2
2275; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
2276; GFX8-NEXT:    v_mov_b32_e32 v0, s5
2277; GFX8-NEXT:    v_mov_b32_e32 v1, s3
2278; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2279; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2280; GFX8-NEXT:    v_mov_b32_e32 v1, s6
2281; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2282; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2283; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2284; GFX8-NEXT:    flat_store_dword v[0:1], v2
2285; GFX8-NEXT:    s_endpgm
2286;
2287; GFX9-NODL-LABEL: udot2_MultipleUses_mul2:
2288; GFX9-NODL:       ; %bb.0: ; %entry
2289; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2290; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2291; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
2292; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2293; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
2294; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
2295; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
2296; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2297; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
2298; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
2299; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
2300; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
2301; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
2302; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
2303; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2304; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2305; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
2306; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2307; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2308; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2309; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2310; GFX9-NODL-NEXT:    s_endpgm
2311;
2312; GFX9-DL-LABEL: udot2_MultipleUses_mul2:
2313; GFX9-DL:       ; %bb.0: ; %entry
2314; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2315; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2316; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
2317; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2318; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
2319; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
2320; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
2321; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2322; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
2323; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
2324; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
2325; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
2326; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
2327; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
2328; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2329; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2330; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
2331; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2332; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2333; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2334; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2335; GFX9-DL-NEXT:    s_endpgm
2336;
2337; GFX10-DL-LABEL: udot2_MultipleUses_mul2:
2338; GFX10-DL:       ; %bb.0: ; %entry
2339; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2340; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2341; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
2342; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2343; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2344; GFX10-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
2345; GFX10-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
2346; GFX10-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
2347; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2348; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2349; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2350; GFX10-DL-NEXT:    s_lshr_b32 s0, s3, 16
2351; GFX10-DL-NEXT:    s_lshr_b32 s1, s4, 16
2352; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s5
2353; GFX10-DL-NEXT:    s_and_b32 s3, s3, s2
2354; GFX10-DL-NEXT:    s_and_b32 s2, s4, s2
2355; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
2356; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v2
2357; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v2
2358; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2359; GFX10-DL-NEXT:    s_endpgm
2360                                                   <2 x i16> addrspace(1)* %src2,
2361                                                   i32 addrspace(1)* nocapture %dst) {
2362entry:
2363  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2364  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2365
2366  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
2367  %conv = zext i16 %s1.elt1 to i32
2368  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
2369  %conv2 = zext i16 %s2.elt1 to i32
2370  %mul1 = mul i32 %conv2, %conv
2371
2372  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
2373  %conv3 = zext i16 %s1.elt2 to i32
2374  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
2375  %conv4 = zext i16 %s2.elt2 to i32
2376  %mul2 = mul i32 %conv4, %conv3
2377
2378  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2379  %add0 = add i32 %mul2, %s3
2380
2381  %add1 = add i32 %mul2, %add0
2382  %add2 = add i32 %add1, %mul1
2383
2384  store i32 %add2, i32 addrspace(1)* %dst, align 4
2385  ret void
2386}
2387
2388define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1,
2389; GFX7-LABEL: idot2_MultipleUses_mul2:
2390; GFX7:       ; %bb.0: ; %entry
2391; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2392; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2393; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2394; GFX7-NEXT:    s_mov_b32 s2, -1
2395; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2396; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2397; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2398; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
2399; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2400; GFX7-NEXT:    s_sext_i32_i16 s7, s4
2401; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
2402; GFX7-NEXT:    s_sext_i32_i16 s8, s5
2403; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
2404; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2405; GFX7-NEXT:    v_mov_b32_e32 v1, s6
2406; GFX7-NEXT:    v_mad_i32_i24 v1, s5, v0, v1
2407; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
2408; GFX7-NEXT:    v_mov_b32_e32 v1, s7
2409; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
2410; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2411; GFX7-NEXT:    s_endpgm
2412;
2413; GFX8-LABEL: idot2_MultipleUses_mul2:
2414; GFX8:       ; %bb.0: ; %entry
2415; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2416; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2417; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2418; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
2419; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
2420; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
2421; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2422; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2423; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2424; GFX8-NEXT:    s_sext_i32_i16 s0, s2
2425; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
2426; GFX8-NEXT:    s_sext_i32_i16 s1, s3
2427; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
2428; GFX8-NEXT:    v_mov_b32_e32 v2, s4
2429; GFX8-NEXT:    v_mov_b32_e32 v3, s2
2430; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
2431; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
2432; GFX8-NEXT:    v_mov_b32_e32 v3, s0
2433; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
2434; GFX8-NEXT:    flat_store_dword v[0:1], v2
2435; GFX8-NEXT:    s_endpgm
2436;
2437; GFX9-NODL-LABEL: idot2_MultipleUses_mul2:
2438; GFX9-NODL:       ; %bb.0: ; %entry
2439; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2440; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2441; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2442; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
2443; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
2444; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
2445; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2446; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2447; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2448; GFX9-NODL-NEXT:    s_sext_i32_i16 s0, s2
2449; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
2450; GFX9-NODL-NEXT:    s_sext_i32_i16 s1, s3
2451; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
2452; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
2453; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s2
2454; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
2455; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
2456; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
2457; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
2458; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2459; GFX9-NODL-NEXT:    s_endpgm
2460;
2461; GFX9-DL-LABEL: idot2_MultipleUses_mul2:
2462; GFX9-DL:       ; %bb.0: ; %entry
2463; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2464; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2465; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2466; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2467; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2468; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
2469; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2470; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2471; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2472; GFX9-DL-NEXT:    s_sext_i32_i16 s0, s2
2473; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
2474; GFX9-DL-NEXT:    s_sext_i32_i16 s1, s3
2475; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
2476; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
2477; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
2478; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
2479; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
2480; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s0
2481; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
2482; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2483; GFX9-DL-NEXT:    s_endpgm
2484;
2485; GFX10-DL-LABEL: idot2_MultipleUses_mul2:
2486; GFX10-DL:       ; %bb.0: ; %entry
2487; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2488; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2489; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2490; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2491; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2492; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2493; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
2494; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2495; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2496; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2497; GFX10-DL-NEXT:    s_ashr_i32 s0, s2, 16
2498; GFX10-DL-NEXT:    s_ashr_i32 s1, s3, 16
2499; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
2500; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s2
2501; GFX10-DL-NEXT:    s_sext_i32_i16 s3, s3
2502; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v2
2503; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v2
2504; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v2
2505; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2506; GFX10-DL-NEXT:    s_endpgm
2507                                                   <2 x i16> addrspace(1)* %src2,
2508                                                   i32 addrspace(1)* nocapture %dst) {
2509entry:
2510  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2511  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2512
2513  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
2514  %conv = sext i16 %s1.elt1 to i32
2515  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
2516  %conv2 = sext i16 %s2.elt1 to i32
2517  %mul1 = mul i32 %conv2, %conv
2518
2519  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
2520  %conv3 = sext i16 %s1.elt2 to i32
2521  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
2522  %conv4 = sext i16 %s2.elt2 to i32
2523  %mul2 = mul i32 %conv4, %conv3
2524
2525  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2526  %add0 = add i32 %mul2, %s3
2527
2528  %add1 = add i32 %mul2, %add0
2529  %add2 = add i32 %add1, %mul1
2530
2531  store i32 %add2, i32 addrspace(1)* %dst, align 4
2532  ret void
2533}
2534
2535define amdgpu_kernel void @udot2_acc16(<2 x i16> addrspace(1)* %src1,
2536; GFX7-LABEL: udot2_acc16:
2537; GFX7:       ; %bb.0: ; %entry
2538; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2539; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2540; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2541; GFX7-NEXT:    s_mov_b32 s2, -1
2542; GFX7-NEXT:    s_mov_b32 s8, 0xffff
2543; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2544; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2545; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
2546; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2547; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2548; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
2549; GFX7-NEXT:    s_and_b32 s4, s4, s8
2550; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
2551; GFX7-NEXT:    v_mov_b32_e32 v1, s7
2552; GFX7-NEXT:    s_and_b32 s5, s5, s8
2553; GFX7-NEXT:    s_waitcnt vmcnt(0)
2554; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
2555; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2556; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2557; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
2558; GFX7-NEXT:    s_endpgm
2559;
2560; GFX8-LABEL: udot2_acc16:
2561; GFX8:       ; %bb.0: ; %entry
2562; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2563; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2564; GFX8-NEXT:    s_mov_b32 s2, 0xffff
2565; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2566; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2567; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2568; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
2569; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
2570; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
2571; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2572; GFX8-NEXT:    s_and_b32 s3, s1, s2
2573; GFX8-NEXT:    s_lshr_b32 s1, s1, 16
2574; GFX8-NEXT:    s_and_b32 s2, s0, s2
2575; GFX8-NEXT:    s_lshr_b32 s0, s0, 16
2576; GFX8-NEXT:    v_mov_b32_e32 v3, s1
2577; GFX8-NEXT:    s_waitcnt vmcnt(0)
2578; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
2579; GFX8-NEXT:    v_mov_b32_e32 v3, s3
2580; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
2581; GFX8-NEXT:    flat_store_short v[0:1], v2
2582; GFX8-NEXT:    s_endpgm
2583;
2584; GFX9-NODL-LABEL: udot2_acc16:
2585; GFX9-NODL:       ; %bb.0: ; %entry
2586; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2587; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2588; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
2589; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2590; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2591; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2592; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
2593; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
2594; GFX9-NODL-NEXT:    s_load_dword s1, s[6:7], 0x0
2595; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2596; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s2
2597; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 16
2598; GFX9-NODL-NEXT:    s_and_b32 s2, s0, s2
2599; GFX9-NODL-NEXT:    s_lshr_b32 s0, s0, 16
2600; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
2601; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
2602; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
2603; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
2604; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
2605; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
2606; GFX9-NODL-NEXT:    s_endpgm
2607;
2608; GFX9-DL-LABEL: udot2_acc16:
2609; GFX9-DL:       ; %bb.0: ; %entry
2610; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2611; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2612; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2613; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2614; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2615; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2616; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2617; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
2618; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2619; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
2620; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2621; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s2, v3, v2
2622; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
2623; GFX9-DL-NEXT:    s_endpgm
2624;
2625; GFX10-DL-LABEL: udot2_acc16:
2626; GFX10-DL:       ; %bb.0: ; %entry
2627; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
2628; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2629; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2630; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
2631; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s3
2632; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2633; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
2634; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2635; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
2636; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
2637; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2638; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s0, s1, v2
2639; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
2640; GFX10-DL-NEXT:    s_endpgm
2641                                       <2 x i16> addrspace(1)* %src2,
2642                                       i16 addrspace(1)* nocapture %dst) {
2643entry:
2644  %v1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2645  %v2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2646
2647  %v1e1 = extractelement <2 x i16> %v1, i64 0
2648  %v2e1 = extractelement <2 x i16> %v2, i64 0
2649  %mul1 = mul i16 %v1e1, %v2e1
2650
2651  %v1e2 = extractelement <2 x i16> %v1, i64 1
2652  %v2e2 = extractelement <2 x i16> %v2, i64 1
2653  %mul2 = mul i16 %v1e2, %v2e2
2654
2655  %s2 = load i16, i16 addrspace(1)* %dst, align 2
2656  %add1 = add i16 %mul2, %s2
2657  %add2 = add i16 %add1, %mul1
2658  store i16 %add2, i16 addrspace(1)* %dst, align 2
2659  ret void
2660}
2661
2662define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
2663; GFX7-LABEL: notsdot2_sext8:
2664; GFX7:       ; %bb.0: ; %entry
2665; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2666; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2667; GFX7-NEXT:    s_mov_b32 s2, -1
2668; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2669; GFX7-NEXT:    s_mov_b32 s10, s2
2670; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2671; GFX7-NEXT:    s_mov_b32 s8, s6
2672; GFX7-NEXT:    s_mov_b32 s9, s7
2673; GFX7-NEXT:    s_mov_b32 s11, s3
2674; GFX7-NEXT:    s_mov_b32 s6, s2
2675; GFX7-NEXT:    s_mov_b32 s7, s3
2676; GFX7-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
2677; GFX7-NEXT:    buffer_load_ushort v1, off, s[8:11], 0
2678; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
2679; GFX7-NEXT:    s_waitcnt vmcnt(1)
2680; GFX7-NEXT:    v_bfe_i32 v2, v0, 0, 8
2681; GFX7-NEXT:    s_waitcnt vmcnt(0)
2682; GFX7-NEXT:    v_bfe_i32 v3, v1, 0, 8
2683; GFX7-NEXT:    v_bfe_i32 v0, v0, 8, 8
2684; GFX7-NEXT:    v_bfe_i32 v1, v1, 8, 8
2685; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2686; GFX7-NEXT:    v_mad_i32_i24 v0, v1, v0, s4
2687; GFX7-NEXT:    v_mad_i32_i24 v0, v3, v2, v0
2688; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2689; GFX7-NEXT:    s_endpgm
2690;
2691; GFX8-LABEL: notsdot2_sext8:
2692; GFX8:       ; %bb.0: ; %entry
2693; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2694; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2695; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2696; GFX8-NEXT:    s_load_dword s2, s[0:1], 0x0
2697; GFX8-NEXT:    v_mov_b32_e32 v0, s6
2698; GFX8-NEXT:    v_mov_b32_e32 v1, s7
2699; GFX8-NEXT:    v_mov_b32_e32 v2, s4
2700; GFX8-NEXT:    v_mov_b32_e32 v3, s5
2701; GFX8-NEXT:    flat_load_ushort v2, v[2:3]
2702; GFX8-NEXT:    flat_load_ushort v3, v[0:1]
2703; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2704; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2705; GFX8-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(0)
2706; GFX8-NEXT:    v_lshrrev_b16_e32 v4, 8, v2
2707; GFX8-NEXT:    s_waitcnt vmcnt(0)
2708; GFX8-NEXT:    v_bfe_i32 v5, v3, 0, 8
2709; GFX8-NEXT:    v_lshrrev_b16_e32 v3, 8, v3
2710; GFX8-NEXT:    v_bfe_i32 v4, v4, 0, 8
2711; GFX8-NEXT:    v_bfe_i32 v3, v3, 0, 8
2712; GFX8-NEXT:    v_bfe_i32 v2, v2, 0, 8
2713; GFX8-NEXT:    v_mad_i32_i24 v3, v3, v4, s2
2714; GFX8-NEXT:    v_mad_i32_i24 v2, v5, v2, v3
2715; GFX8-NEXT:    flat_store_dword v[0:1], v2
2716; GFX8-NEXT:    s_endpgm
2717;
2718; GFX9-NODL-LABEL: notsdot2_sext8:
2719; GFX9-NODL:       ; %bb.0: ; %entry
2720; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2721; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2722; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2723; GFX9-NODL-NEXT:    s_load_dword s2, s[0:1], 0x0
2724; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s6
2725; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s7
2726; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s4
2727; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s5
2728; GFX9-NODL-NEXT:    global_load_ushort v2, v[2:3], off
2729; GFX9-NODL-NEXT:    global_load_ushort v3, v[0:1], off
2730; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2731; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2732; GFX9-NODL-NEXT:    s_waitcnt vmcnt(1)
2733; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v4, 8, v2
2734; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
2735; GFX9-NODL-NEXT:    v_bfe_i32 v5, v3, 0, 8
2736; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v3, 8, v3
2737; GFX9-NODL-NEXT:    v_bfe_i32 v4, v4, 0, 8
2738; GFX9-NODL-NEXT:    v_bfe_i32 v3, v3, 0, 8
2739; GFX9-NODL-NEXT:    v_bfe_i32 v2, v2, 0, 8
2740; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2741; GFX9-NODL-NEXT:    v_mad_i32_i24 v3, v3, v4, s2
2742; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, v5, v2, v3
2743; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2744; GFX9-NODL-NEXT:    s_endpgm
2745;
2746; GFX9-DL-LABEL: notsdot2_sext8:
2747; GFX9-DL:       ; %bb.0: ; %entry
2748; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2749; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2750; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2751; GFX9-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
2752; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
2753; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s7
2754; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
2755; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
2756; GFX9-DL-NEXT:    global_load_ushort v2, v[2:3], off
2757; GFX9-DL-NEXT:    global_load_ushort v3, v[0:1], off
2758; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2759; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2760; GFX9-DL-NEXT:    s_waitcnt vmcnt(1)
2761; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v4, 8, v2
2762; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2763; GFX9-DL-NEXT:    v_bfe_i32 v5, v3, 0, 8
2764; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v3, 8, v3
2765; GFX9-DL-NEXT:    v_bfe_i32 v4, v4, 0, 8
2766; GFX9-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
2767; GFX9-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
2768; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2769; GFX9-DL-NEXT:    v_mad_i32_i24 v3, v3, v4, s2
2770; GFX9-DL-NEXT:    v_mad_i32_i24 v2, v5, v2, v3
2771; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2772; GFX9-DL-NEXT:    s_endpgm
2773;
2774; GFX10-DL-LABEL: notsdot2_sext8:
2775; GFX10-DL:       ; %bb.0: ; %entry
2776; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2777; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2778; GFX10-DL-NEXT:    v_mov_b32_e32 v4, 0xffff
2779; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2780; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2781; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
2782; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s4
2783; GFX10-DL-NEXT:    v_mov_b32_e32 v3, s5
2784; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s7
2785; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
2786; GFX10-DL-NEXT:    global_load_ushort v2, v[2:3], off
2787; GFX10-DL-NEXT:    global_load_ushort v7, v[0:1], off
2788; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
2789; GFX10-DL-NEXT:    v_and_b32_sdwa v1, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
2790; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
2791; GFX10-DL-NEXT:    v_and_b32_sdwa v3, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
2792; GFX10-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
2793; GFX10-DL-NEXT:    v_bfe_i32 v0, v7, 0, 8
2794; GFX10-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
2795; GFX10-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
2796; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2797; GFX10-DL-NEXT:    v_mad_i32_i24 v1, v3, v1, s2
2798; GFX10-DL-NEXT:    v_mad_i32_i24 v2, v0, v2, v1
2799; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2800; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2801; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2802; GFX10-DL-NEXT:    s_endpgm
2803                                          <2 x i8> addrspace(1)* %src2,
2804                                          i32 addrspace(1)* nocapture %dst) {
2805entry:
2806  %vec1 = load <2 x i8>, <2 x i8> addrspace(1)* %src1
2807  %vec2 = load <2 x i8>, <2 x i8> addrspace(1)* %src2
2808
2809  %s1.elt1 = extractelement <2 x i8> %vec1, i64 0
2810  %conv = sext i8 %s1.elt1 to i32
2811  %s2.elt1 = extractelement <2 x i8> %vec2, i64 0
2812  %conv2 = sext i8 %s2.elt1 to i32
2813  %mul1 = mul nuw i32 %conv2, %conv
2814
2815  %s1.elt2 = extractelement <2 x i8> %vec1, i64 1
2816  %conv3 = sext i8 %s1.elt2 to i32
2817  %s2.elt2 = extractelement <2 x i8> %vec2, i64 1
2818  %conv4 = sext i8 %s2.elt2 to i32
2819  %mul2 = mul nuw i32 %conv4, %conv3
2820
2821  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2822  %add = add i32 %mul2, %s3
2823  %add6 = add i32 %add, %mul1
2824  store i32 %add6, i32 addrspace(1)* %dst, align 4
2825  ret void
2826}
2827