1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX8 %s
4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-NODL %s
5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-DL %s
6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-DL %s
7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-DL %s
8
9; add(mul(S0.x, S1.y),
10;     add (mul (S0.y, S1.y), S3)) -> v_dot2_{I|U}32_{I|U}16(S1, S2, S3)
11
12define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1,
13; GFX7-LABEL: udot2:
14; GFX7:       ; %bb.0: ; %entry
15; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
16; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
17; GFX7-NEXT:    s_mov_b32 s8, 0xffff
18; GFX7-NEXT:    s_mov_b32 s3, 0xf000
19; GFX7-NEXT:    s_mov_b32 s2, -1
20; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
22; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
23; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
24; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
25; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
26; GFX7-NEXT:    s_and_b32 s4, s4, s8
27; GFX7-NEXT:    s_and_b32 s5, s5, s8
28; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
29; GFX7-NEXT:    v_mov_b32_e32 v0, s6
30; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
31; GFX7-NEXT:    v_mov_b32_e32 v1, s8
32; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
33; GFX7-NEXT:    v_mov_b32_e32 v1, s4
34; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
35; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
36; GFX7-NEXT:    s_endpgm
37;
38; GFX8-LABEL: udot2:
39; GFX8:       ; %bb.0: ; %entry
40; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
41; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
42; GFX8-NEXT:    s_mov_b32 s2, 0xffff
43; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
45; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
46; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
47; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
48; GFX8-NEXT:    s_and_b32 s6, s3, s2
49; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
50; GFX8-NEXT:    s_and_b32 s2, s4, s2
51; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
52; GFX8-NEXT:    v_mov_b32_e32 v0, s5
53; GFX8-NEXT:    v_mov_b32_e32 v1, s3
54; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
55; GFX8-NEXT:    v_mov_b32_e32 v1, s6
56; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
57; GFX8-NEXT:    v_mov_b32_e32 v0, s0
58; GFX8-NEXT:    v_mov_b32_e32 v1, s1
59; GFX8-NEXT:    flat_store_dword v[0:1], v2
60; GFX8-NEXT:    s_endpgm
61;
62; GFX9-NODL-LABEL: udot2:
63; GFX9-NODL:       ; %bb.0: ; %entry
64; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
65; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
66; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
67; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
68; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
69; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
70; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
71; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
72; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
73; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
74; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
75; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
76; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
77; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
78; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
79; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
80; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
81; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
82; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
83; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
84; GFX9-NODL-NEXT:    s_endpgm
85;
86; GFX9-DL-LABEL: udot2:
87; GFX9-DL:       ; %bb.0: ; %entry
88; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
89; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
90; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
91; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
92; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
93; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
94; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
95; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
96; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
97; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s4, v0, v1
98; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
99; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
100; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
101; GFX9-DL-NEXT:    s_endpgm
102;
103; GFX10-DL-LABEL: udot2:
104; GFX10-DL:       ; %bb.0: ; %entry
105; GFX10-DL-NEXT:    s_clause 0x1
106; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
107; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
108; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
109; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
110; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
111; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
112; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
113; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
114; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
115; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s1, s0, v0
116; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
117; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
118; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
119; GFX10-DL-NEXT:    s_endpgm
120                                 <2 x i16> addrspace(1)* %src2,
121                                 i32 addrspace(1)* nocapture %dst) {
122entry:
123  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
124  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
125
126  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
127  %conv = zext i16 %s1.elt1 to i32
128  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
129  %conv2 = zext i16 %s2.elt1 to i32
130  %mul1 = mul nuw i32 %conv2, %conv
131
132  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
133  %conv3 = zext i16 %s1.elt2 to i32
134  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
135  %conv4 = zext i16 %s2.elt2 to i32
136  %mul2 = mul nuw i32 %conv4, %conv3
137
138  %s3 = load i32, i32 addrspace(1)* %dst, align 4
139  %add = add i32 %mul2, %s3
140  %add6 = add i32 %add, %mul1
141  store i32 %add6, i32 addrspace(1)* %dst, align 4
142  ret void
143}
144
145; TODO: Support this pattern
146;      add(S3,
147;          add (mul (S0.y, S1.y), mul (S0.y, S1.y))) -> v_dot2_{I|U}32_{I|U}16(S1, S2, S3)
148define amdgpu_kernel void @udot2_MulMul(<2 x i16> addrspace(1)* %src1,
149; GFX7-LABEL: udot2_MulMul:
150; GFX7:       ; %bb.0: ; %entry
151; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
152; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
153; GFX7-NEXT:    s_mov_b32 s8, 0xffff
154; GFX7-NEXT:    s_mov_b32 s3, 0xf000
155; GFX7-NEXT:    s_mov_b32 s2, -1
156; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
157; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
158; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
159; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
160; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
161; GFX7-NEXT:    s_and_b32 s4, s4, s8
162; GFX7-NEXT:    v_mov_b32_e32 v0, s4
163; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x0
164; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
165; GFX7-NEXT:    s_and_b32 s5, s5, s8
166; GFX7-NEXT:    v_mul_u32_u24_e32 v0, s5, v0
167; GFX7-NEXT:    v_mov_b32_e32 v1, s6
168; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
169; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
170; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
171; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
172; GFX7-NEXT:    s_endpgm
173;
174; GFX8-LABEL: udot2_MulMul:
175; GFX8:       ; %bb.0: ; %entry
176; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
177; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
178; GFX8-NEXT:    s_mov_b32 s2, 0xffff
179; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
180; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
181; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
182; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
183; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
184; GFX8-NEXT:    s_and_b32 s6, s3, s2
185; GFX8-NEXT:    s_and_b32 s2, s4, s2
186; GFX8-NEXT:    v_mov_b32_e32 v0, s6
187; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
188; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
189; GFX8-NEXT:    v_mov_b32_e32 v1, s3
190; GFX8-NEXT:    v_mul_u32_u24_e32 v0, s2, v0
191; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
192; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s5, v0
193; GFX8-NEXT:    v_mov_b32_e32 v0, s0
194; GFX8-NEXT:    v_mov_b32_e32 v1, s1
195; GFX8-NEXT:    flat_store_dword v[0:1], v2
196; GFX8-NEXT:    s_endpgm
197;
198; GFX9-NODL-LABEL: udot2_MulMul:
199; GFX9-NODL:       ; %bb.0: ; %entry
200; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
201; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
202; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
203; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
204; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
205; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
206; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
207; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
208; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
209; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
210; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s6
211; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
212; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
213; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
214; GFX9-NODL-NEXT:    v_mul_u32_u24_e32 v0, s2, v0
215; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
216; GFX9-NODL-NEXT:    v_add_u32_e32 v2, s5, v0
217; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
218; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
219; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
220; GFX9-NODL-NEXT:    s_endpgm
221;
222; GFX9-DL-LABEL: udot2_MulMul:
223; GFX9-DL:       ; %bb.0: ; %entry
224; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
225; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
226; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
227; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
228; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
229; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
230; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
231; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
232; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
233; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
234; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
235; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
236; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
237; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
238; GFX9-DL-NEXT:    v_mul_u32_u24_e32 v0, s2, v0
239; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
240; GFX9-DL-NEXT:    v_add_u32_e32 v2, s5, v0
241; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
242; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
243; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
244; GFX9-DL-NEXT:    s_endpgm
245;
246; GFX10-DL-LABEL: udot2_MulMul:
247; GFX10-DL:       ; %bb.0: ; %entry
248; GFX10-DL-NEXT:    s_clause 0x1
249; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
250; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
251; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
252; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
253; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
254; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
255; GFX10-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
256; GFX10-DL-NEXT:    s_mov_b32 s5, 0xffff
257; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
258; GFX10-DL-NEXT:    s_and_b32 s6, s2, s5
259; GFX10-DL-NEXT:    s_and_b32 s5, s3, s5
260; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 16
261; GFX10-DL-NEXT:    v_mul_u32_u24_e64 v0, s5, s6
262; GFX10-DL-NEXT:    s_lshr_b32 s3, s3, 16
263; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s3, s2, v0
264; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, s4, v0
265; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
266; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
267; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
268; GFX10-DL-NEXT:    s_endpgm
269                                        <2 x i16> addrspace(1)* %src2,
270                                        i32 addrspace(1)* nocapture %dst) {
271entry:
272  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
273  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
274
275  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
276  %conv = zext i16 %s1.elt1 to i32
277  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
278  %conv2 = zext i16 %s2.elt1 to i32
279  %mul1 = mul nuw i32 %conv2, %conv
280
281  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
282  %conv3 = zext i16 %s1.elt2 to i32
283  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
284  %conv4 = zext i16 %s2.elt2 to i32
285  %mul2 = mul nuw i32 %conv4, %conv3
286  %s3 = load i32, i32 addrspace(1)* %dst, align 4
287  %add = add i32 %mul2, %mul1
288  %add6 = add i32 %add, %s3
289  store i32 %add6, i32 addrspace(1)* %dst, align 4
290  ret void
291}
292
293define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
294; GFX7-LABEL: idot2:
295; GFX7:       ; %bb.0: ; %entry
296; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
297; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
298; GFX7-NEXT:    s_mov_b32 s3, 0xf000
299; GFX7-NEXT:    s_mov_b32 s2, -1
300; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
301; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
302; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
303; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
304; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
305; GFX7-NEXT:    s_sext_i32_i16 s7, s4
306; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
307; GFX7-NEXT:    s_sext_i32_i16 s8, s5
308; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
309; GFX7-NEXT:    v_mov_b32_e32 v0, s4
310; GFX7-NEXT:    v_mov_b32_e32 v1, s6
311; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
312; GFX7-NEXT:    v_mov_b32_e32 v1, s7
313; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
314; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
315; GFX7-NEXT:    s_endpgm
316;
317; GFX8-LABEL: idot2:
318; GFX8:       ; %bb.0: ; %entry
319; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
320; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
321; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
322; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
323; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
324; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
325; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
326; GFX8-NEXT:    s_sext_i32_i16 s5, s2
327; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
328; GFX8-NEXT:    s_sext_i32_i16 s6, s3
329; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
330; GFX8-NEXT:    v_mov_b32_e32 v0, s4
331; GFX8-NEXT:    v_mov_b32_e32 v1, s2
332; GFX8-NEXT:    v_mov_b32_e32 v2, s5
333; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
334; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
335; GFX8-NEXT:    v_mov_b32_e32 v0, s0
336; GFX8-NEXT:    v_mov_b32_e32 v1, s1
337; GFX8-NEXT:    flat_store_dword v[0:1], v2
338; GFX8-NEXT:    s_endpgm
339;
340; GFX9-NODL-LABEL: idot2:
341; GFX9-NODL:       ; %bb.0: ; %entry
342; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
343; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
344; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
345; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
346; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
347; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
348; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
349; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
350; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
351; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
352; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
353; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
354; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
355; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
356; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
357; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
358; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
359; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
360; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
361; GFX9-NODL-NEXT:    s_endpgm
362;
363; GFX9-DL-LABEL: idot2:
364; GFX9-DL:       ; %bb.0: ; %entry
365; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
366; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
367; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
368; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
369; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
370; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
371; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
372; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
373; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
374; GFX9-DL-NEXT:    v_dot2_i32_i16 v2, s4, v0, v1
375; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
376; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
377; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
378; GFX9-DL-NEXT:    s_endpgm
379;
380; GFX10-DL-LABEL: idot2:
381; GFX10-DL:       ; %bb.0: ; %entry
382; GFX10-DL-NEXT:    s_clause 0x1
383; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
384; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
385; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
386; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
387; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
388; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
389; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
390; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
391; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
392; GFX10-DL-NEXT:    v_dot2_i32_i16 v2, s1, s0, v0
393; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
394; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
395; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
396; GFX10-DL-NEXT:    s_endpgm
397                                 <2 x i16> addrspace(1)* %src2,
398                                 i32 addrspace(1)* nocapture %dst) {
399entry:
400  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
401  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
402
403  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
404  %conv = sext i16 %s1.elt1 to i32
405  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
406  %conv2 = sext i16 %s2.elt1 to i32
407  %mul1 = mul nuw i32 %conv2, %conv
408
409  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
410  %conv3 = sext i16 %s1.elt2 to i32
411  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
412  %conv4 = sext i16 %s2.elt2 to i32
413  %mul2 = mul nuw i32 %conv4, %conv3
414
415  %s3 = load i32, i32 addrspace(1)* %dst, align 4
416  %add = add i32 %mul2, %s3
417  %add6 = add i32 %add, %mul1
418  store i32 %add6, i32 addrspace(1)* %dst, align 4
419  ret void
420}
421
422define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
423; GFX7-LABEL: idot2_MixedTypedMul:
424; GFX7:       ; %bb.0: ; %entry
425; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
426; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
427; GFX7-NEXT:    s_mov_b32 s3, 0xf000
428; GFX7-NEXT:    s_mov_b32 s2, -1
429; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
430; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
431; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
432; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
433; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
434; GFX7-NEXT:    s_lshr_b32 s7, s4, 16
435; GFX7-NEXT:    s_lshr_b32 s8, s5, 16
436; GFX7-NEXT:    s_sext_i32_i16 s4, s4
437; GFX7-NEXT:    v_mov_b32_e32 v0, s7
438; GFX7-NEXT:    v_mov_b32_e32 v1, s6
439; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v0, v1
440; GFX7-NEXT:    s_sext_i32_i16 s5, s5
441; GFX7-NEXT:    v_mov_b32_e32 v1, s4
442; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v1, v0
443; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
444; GFX7-NEXT:    s_endpgm
445;
446; GFX8-LABEL: idot2_MixedTypedMul:
447; GFX8:       ; %bb.0: ; %entry
448; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
449; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
450; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
451; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
452; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
453; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
454; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
455; GFX8-NEXT:    s_sext_i32_i16 s5, s2
456; GFX8-NEXT:    s_lshr_b32 s2, s2, 16
457; GFX8-NEXT:    s_sext_i32_i16 s6, s3
458; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
459; GFX8-NEXT:    v_mov_b32_e32 v0, s4
460; GFX8-NEXT:    v_mov_b32_e32 v1, s2
461; GFX8-NEXT:    v_mov_b32_e32 v2, s5
462; GFX8-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
463; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
464; GFX8-NEXT:    v_mov_b32_e32 v0, s0
465; GFX8-NEXT:    v_mov_b32_e32 v1, s1
466; GFX8-NEXT:    flat_store_dword v[0:1], v2
467; GFX8-NEXT:    s_endpgm
468;
469; GFX9-NODL-LABEL: idot2_MixedTypedMul:
470; GFX9-NODL:       ; %bb.0: ; %entry
471; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
472; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
473; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
474; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
475; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
476; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
477; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
478; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
479; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 16
480; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
481; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
482; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
483; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
484; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
485; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
486; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
487; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
488; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
489; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
490; GFX9-NODL-NEXT:    s_endpgm
491;
492; GFX9-DL-LABEL: idot2_MixedTypedMul:
493; GFX9-DL:       ; %bb.0: ; %entry
494; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
495; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
496; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
497; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
498; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
499; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
500; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
501; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
502; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 16
503; GFX9-DL-NEXT:    s_sext_i32_i16 s6, s3
504; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
505; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
506; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
507; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
508; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s3, v1, v0
509; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
510; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
511; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
512; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
513; GFX9-DL-NEXT:    s_endpgm
514;
515; GFX10-DL-LABEL: idot2_MixedTypedMul:
516; GFX10-DL:       ; %bb.0: ; %entry
517; GFX10-DL-NEXT:    s_clause 0x1
518; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
519; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
520; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
521; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
522; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
523; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
524; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
525; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
526; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
527; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 16
528; GFX10-DL-NEXT:    s_lshr_b32 s3, s1, 16
529; GFX10-DL-NEXT:    s_sext_i32_i16 s0, s0
530; GFX10-DL-NEXT:    s_sext_i32_i16 s1, s1
531; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s3, s2, v0
532; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v0
533; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
534; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
535; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
536; GFX10-DL-NEXT:    s_endpgm
537                                               <2 x i16> addrspace(1)* %src2,
538                                               i32 addrspace(1)* nocapture %dst) {
539entry:
540  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
541  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
542
543  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
544  %conv = sext i16 %s1.elt1 to i32
545  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
546  %conv2 = sext i16 %s2.elt1 to i32
547  %mul1 = mul nuw i32 %conv2, %conv
548
549  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
550  %conv3 = zext i16 %s1.elt2 to i32
551  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
552  %conv4 = zext i16 %s2.elt2 to i32
553  %mul2 = mul nuw i32 %conv4, %conv3
554
555  %s3 = load i32, i32 addrspace(1)* %dst, align 4
556  %add = add i32 %mul2, %s3
557  %add6 = add i32 %add, %mul1
558  store i32 %add6, i32 addrspace(1)* %dst, align 4
559  ret void
560}
561
562define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1,
563; GFX7-LABEL: udot2_alt_AddOperands:
564; GFX7:       ; %bb.0: ; %entry
565; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
566; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
567; GFX7-NEXT:    s_mov_b32 s8, 0xffff
568; GFX7-NEXT:    s_mov_b32 s3, 0xf000
569; GFX7-NEXT:    s_mov_b32 s2, -1
570; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
571; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
572; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
573; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
574; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
575; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
576; GFX7-NEXT:    s_and_b32 s4, s4, s8
577; GFX7-NEXT:    s_and_b32 s5, s5, s8
578; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
579; GFX7-NEXT:    v_mov_b32_e32 v0, s6
580; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
581; GFX7-NEXT:    v_mov_b32_e32 v1, s8
582; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
583; GFX7-NEXT:    v_mov_b32_e32 v1, s4
584; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
585; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
586; GFX7-NEXT:    s_endpgm
587;
588; GFX8-LABEL: udot2_alt_AddOperands:
589; GFX8:       ; %bb.0: ; %entry
590; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
591; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
592; GFX8-NEXT:    s_mov_b32 s2, 0xffff
593; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
594; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
595; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
596; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
597; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
598; GFX8-NEXT:    s_and_b32 s6, s3, s2
599; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
600; GFX8-NEXT:    s_and_b32 s2, s4, s2
601; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
602; GFX8-NEXT:    v_mov_b32_e32 v0, s5
603; GFX8-NEXT:    v_mov_b32_e32 v1, s3
604; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
605; GFX8-NEXT:    v_mov_b32_e32 v1, s6
606; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
607; GFX8-NEXT:    v_mov_b32_e32 v0, s0
608; GFX8-NEXT:    v_mov_b32_e32 v1, s1
609; GFX8-NEXT:    flat_store_dword v[0:1], v2
610; GFX8-NEXT:    s_endpgm
611;
612; GFX9-NODL-LABEL: udot2_alt_AddOperands:
613; GFX9-NODL:       ; %bb.0: ; %entry
614; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
615; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
616; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
617; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
618; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
619; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
620; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
621; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
622; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
623; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
624; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
625; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
626; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
627; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
628; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
629; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
630; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
631; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
632; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
633; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
634; GFX9-NODL-NEXT:    s_endpgm
635;
636; GFX9-DL-LABEL: udot2_alt_AddOperands:
637; GFX9-DL:       ; %bb.0: ; %entry
638; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
639; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
640; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
641; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
642; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
643; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
644; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
645; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
646; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
647; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s4, v0, v1
648; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
649; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
650; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
651; GFX9-DL-NEXT:    s_endpgm
652;
653; GFX10-DL-LABEL: udot2_alt_AddOperands:
654; GFX10-DL:       ; %bb.0: ; %entry
655; GFX10-DL-NEXT:    s_clause 0x1
656; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
657; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
658; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
659; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
660; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
661; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
662; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
663; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
664; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
665; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s1, s0, v0
666; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
667; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
668; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
669; GFX10-DL-NEXT:    s_endpgm
670                                                 <2 x i16> addrspace(1)* %src2,
671                                                 i32 addrspace(1)* nocapture %dst) {
672entry:
673  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
674  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
675
676  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
677  %conv = zext i16 %s1.elt1 to i32
678  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
679  %conv2 = zext i16 %s2.elt1 to i32
680  %mul1 = mul nuw i32 %conv2, %conv
681
682  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
683  %conv3 = zext i16 %s1.elt2 to i32
684  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
685  %conv4 = zext i16 %s2.elt2 to i32
686  %mul2 = mul nuw i32 %conv4, %conv3
687
688  %s3 = load i32, i32 addrspace(1)* %dst, align 4
689  %add = add i32 %s3, %mul2
690  %add6 = add i32 %mul1, %add
691  store i32 %add6, i32 addrspace(1)* %dst, align 4
692  ret void
693}
694
695define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
696; GFX7-LABEL: idot2_MixedExt:
697; GFX7:       ; %bb.0: ; %entry
698; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
699; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
700; GFX7-NEXT:    s_mov_b32 s3, 0xf000
701; GFX7-NEXT:    s_mov_b32 s2, -1
702; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
703; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
704; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
705; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
706; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
707; GFX7-NEXT:    s_sext_i32_i16 s7, s4
708; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
709; GFX7-NEXT:    s_and_b32 s8, s5, 0xffff
710; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
711; GFX7-NEXT:    v_mov_b32_e32 v0, s4
712; GFX7-NEXT:    v_mov_b32_e32 v1, s6
713; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
714; GFX7-NEXT:    v_mov_b32_e32 v1, s7
715; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
716; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
717; GFX7-NEXT:    s_endpgm
718;
719; GFX8-LABEL: idot2_MixedExt:
720; GFX8:       ; %bb.0: ; %entry
721; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
722; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
723; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
724; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
725; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
726; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
727; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
728; GFX8-NEXT:    s_sext_i32_i16 s5, s2
729; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
730; GFX8-NEXT:    s_and_b32 s6, s3, 0xffff
731; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
732; GFX8-NEXT:    v_mov_b32_e32 v0, s4
733; GFX8-NEXT:    v_mov_b32_e32 v1, s2
734; GFX8-NEXT:    v_mov_b32_e32 v2, s5
735; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
736; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
737; GFX8-NEXT:    v_mov_b32_e32 v0, s0
738; GFX8-NEXT:    v_mov_b32_e32 v1, s1
739; GFX8-NEXT:    flat_store_dword v[0:1], v2
740; GFX8-NEXT:    s_endpgm
741;
742; GFX9-NODL-LABEL: idot2_MixedExt:
743; GFX9-NODL:       ; %bb.0: ; %entry
744; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
745; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
746; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
747; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
748; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
749; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
750; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
751; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
752; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
753; GFX9-NODL-NEXT:    s_and_b32 s6, s3, 0xffff
754; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
755; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
756; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
757; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
758; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
759; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
760; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
761; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
762; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
763; GFX9-NODL-NEXT:    s_endpgm
764;
765; GFX9-DL-LABEL: idot2_MixedExt:
766; GFX9-DL:       ; %bb.0: ; %entry
767; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
768; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
769; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
770; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
771; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
772; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
773; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
774; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
775; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
776; GFX9-DL-NEXT:    s_and_b32 s6, s3, 0xffff
777; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
778; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
779; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
780; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
781; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
782; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
783; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
784; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
785; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
786; GFX9-DL-NEXT:    s_endpgm
787;
788; GFX10-DL-LABEL: idot2_MixedExt:
789; GFX10-DL:       ; %bb.0: ; %entry
790; GFX10-DL-NEXT:    s_clause 0x1
791; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
792; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
793; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
794; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
795; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
796; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
797; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
798; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
799; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
800; GFX10-DL-NEXT:    s_ashr_i32 s2, s0, 16
801; GFX10-DL-NEXT:    s_ashr_i32 s3, s1, 16
802; GFX10-DL-NEXT:    s_sext_i32_i16 s0, s0
803; GFX10-DL-NEXT:    s_and_b32 s1, s1, 0xffff
804; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s3, s2, v0
805; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v0
806; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
807; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
808; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
809; GFX10-DL-NEXT:    s_endpgm
810                                          <2 x i16> addrspace(1)* %src2,
811                                          i32 addrspace(1)* nocapture %dst) {
812entry:
813  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
814  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
815
816  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
817  %conv = sext i16 %s1.elt1 to i32
818  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
819  %conv2 = zext i16 %s2.elt1 to i32
820  %mul1 = mul nuw i32 %conv2, %conv
821
822  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
823  %conv3 = sext i16 %s1.elt2 to i32
824  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
825  %conv4 = sext i16 %s2.elt2 to i32
826  %mul2 = mul nuw i32 %conv4, %conv3
827
828  %s3 = load i32, i32 addrspace(1)* %dst, align 4
829  %add = add i32 %mul2, %s3
830  %add6 = add i32 %add, %mul1
831  store i32 %add6, i32 addrspace(1)* %dst, align 4
832  ret void
833}
834
835define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
836; GFX7-LABEL: notudot2_SameVec:
837; GFX7:       ; %bb.0: ; %entry
838; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
839; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
840; GFX7-NEXT:    s_mov_b32 s3, 0xf000
841; GFX7-NEXT:    s_mov_b32 s2, -1
842; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
843; GFX7-NEXT:    s_load_dword s6, s[6:7], 0x0
844; GFX7-NEXT:    s_load_dword s7, s[0:1], 0x0
845; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
846; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
847; GFX7-NEXT:    s_lshr_b32 s5, s6, 16
848; GFX7-NEXT:    v_mov_b32_e32 v0, s7
849; GFX7-NEXT:    s_and_b32 s4, s4, 0xffff
850; GFX7-NEXT:    v_mad_u32_u24 v0, s5, s5, v0
851; GFX7-NEXT:    v_mad_u32_u24 v0, s4, s4, v0
852; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
853; GFX7-NEXT:    s_endpgm
854;
855; GFX8-LABEL: notudot2_SameVec:
856; GFX8:       ; %bb.0: ; %entry
857; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
858; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
859; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
860; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
861; GFX8-NEXT:    s_load_dword s3, s[0:1], 0x0
862; GFX8-NEXT:    s_load_dword s4, s[4:5], 0x0
863; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
864; GFX8-NEXT:    s_lshr_b32 s2, s2, 16
865; GFX8-NEXT:    v_mov_b32_e32 v0, s3
866; GFX8-NEXT:    s_and_b32 s4, s4, 0xffff
867; GFX8-NEXT:    v_mad_u32_u24 v0, s2, s2, v0
868; GFX8-NEXT:    v_mad_u32_u24 v2, s4, s4, v0
869; GFX8-NEXT:    v_mov_b32_e32 v0, s0
870; GFX8-NEXT:    v_mov_b32_e32 v1, s1
871; GFX8-NEXT:    flat_store_dword v[0:1], v2
872; GFX8-NEXT:    s_endpgm
873;
874; GFX9-NODL-LABEL: notudot2_SameVec:
875; GFX9-NODL:       ; %bb.0: ; %entry
876; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
877; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
878; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
879; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
880; GFX9-NODL-NEXT:    s_load_dword s3, s[0:1], 0x0
881; GFX9-NODL-NEXT:    s_load_dword s4, s[4:5], 0x0
882; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
883; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 16
884; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s3
885; GFX9-NODL-NEXT:    s_and_b32 s4, s4, 0xffff
886; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s2, s2, v0
887; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, s4, v0
888; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
889; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
890; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
891; GFX9-NODL-NEXT:    s_endpgm
892;
893; GFX9-DL-LABEL: notudot2_SameVec:
894; GFX9-DL:       ; %bb.0: ; %entry
895; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
896; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
897; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
898; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
899; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
900; GFX9-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
901; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
902; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 16
903; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s3
904; GFX9-DL-NEXT:    s_and_b32 s4, s4, 0xffff
905; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s2, s2, v0
906; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, s4, v0
907; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
908; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
909; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
910; GFX9-DL-NEXT:    s_endpgm
911;
912; GFX10-DL-LABEL: notudot2_SameVec:
913; GFX10-DL:       ; %bb.0: ; %entry
914; GFX10-DL-NEXT:    s_clause 0x1
915; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
916; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
917; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
918; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
919; GFX10-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
920; GFX10-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
921; GFX10-DL-NEXT:    s_load_dword s4, s[4:5], 0x0
922; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
923; GFX10-DL-NEXT:    s_lshr_b32 s2, s2, 16
924; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s2, s2, s3
925; GFX10-DL-NEXT:    s_and_b32 s2, s4, 0xffff
926; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s2, v0
927; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
928; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
929; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
930; GFX10-DL-NEXT:    s_endpgm
931                                            <2 x i16> addrspace(1)* %src2,
932                                            i32 addrspace(1)* nocapture %dst) {
933entry:
934  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
935  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
936
937  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
938  %conv = zext i16 %s1.elt1 to i32
939  %s2.elt1 = extractelement <2 x i16> %vec1, i64 0
940  %conv2 = zext i16 %s2.elt1 to i32
941  %mul1 = mul i32 %conv2, %conv
942
943  %s1.elt2 = extractelement <2 x i16> %vec2, i64 1
944  %conv3 = zext i16 %s1.elt2 to i32
945  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
946  %conv4 = zext i16 %s2.elt2 to i32
947  %mul2 = mul i32 %conv4, %conv3
948
949  %s3 = load i32, i32 addrspace(1)* %dst, align 4
950  %add = add i32 %mul2, %s3
951  %add6 = add i32 %add, %mul1
952  store i32 %add6, i32 addrspace(1)* %dst, align 4
953  ret void
954}
955
956define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1,
957; GFX7-LABEL: udot2_v4i16:
958; GFX7:       ; %bb.0: ; %entry
959; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
960; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
961; GFX7-NEXT:    s_mov_b32 s8, 0xffff
962; GFX7-NEXT:    s_mov_b32 s3, 0xf000
963; GFX7-NEXT:    s_mov_b32 s2, -1
964; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
965; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
966; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
967; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
968; GFX7-NEXT:    s_and_b32 s6, s4, s8
969; GFX7-NEXT:    s_and_b32 s7, s5, s8
970; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
971; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
972; GFX7-NEXT:    s_lshr_b32 s5, s5, 16
973; GFX7-NEXT:    v_mov_b32_e32 v0, s4
974; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
975; GFX7-NEXT:    v_mov_b32_e32 v1, s8
976; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
977; GFX7-NEXT:    v_mov_b32_e32 v1, s6
978; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
979; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
980; GFX7-NEXT:    s_endpgm
981;
982; GFX8-LABEL: udot2_v4i16:
983; GFX8:       ; %bb.0: ; %entry
984; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
985; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
986; GFX8-NEXT:    s_mov_b32 s2, 0xffff
987; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
988; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
989; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
990; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
991; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
992; GFX8-NEXT:    s_and_b32 s6, s3, s2
993; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
994; GFX8-NEXT:    s_and_b32 s2, s4, s2
995; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
996; GFX8-NEXT:    v_mov_b32_e32 v0, s5
997; GFX8-NEXT:    v_mov_b32_e32 v1, s3
998; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
999; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1000; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
1001; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1002; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1003; GFX8-NEXT:    flat_store_dword v[0:1], v2
1004; GFX8-NEXT:    s_endpgm
1005;
1006; GFX9-NODL-LABEL: udot2_v4i16:
1007; GFX9-NODL:       ; %bb.0: ; %entry
1008; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1009; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1010; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1011; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1012; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1013; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1014; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1015; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1016; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1017; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
1018; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1019; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
1020; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
1021; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1022; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1023; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1024; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
1025; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1026; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1027; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1028; GFX9-NODL-NEXT:    s_endpgm
1029;
1030; GFX9-DL-LABEL: udot2_v4i16:
1031; GFX9-DL:       ; %bb.0: ; %entry
1032; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1033; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1034; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1035; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1036; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
1037; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1038; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1039; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
1040; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1041; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s4, v0, v1
1042; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1043; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1044; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1045; GFX9-DL-NEXT:    s_endpgm
1046;
1047; GFX10-DL-LABEL: udot2_v4i16:
1048; GFX10-DL:       ; %bb.0: ; %entry
1049; GFX10-DL-NEXT:    s_clause 0x1
1050; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1051; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1052; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1053; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1054; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
1055; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
1056; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
1057; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1058; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1059; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s1, s0, v0
1060; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
1061; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
1062; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1063; GFX10-DL-NEXT:    s_endpgm
1064                                       <4 x i16> addrspace(1)* %src2,
1065                                       i32 addrspace(1)* nocapture %dst) {
1066entry:
1067  %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
1068  %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
1069
1070  %s1.elt1 = extractelement <4 x i16> %vec1, i64 0
1071  %conv = zext i16 %s1.elt1 to i32
1072  %s2.elt1 = extractelement <4 x i16> %vec2, i64 0
1073  %conv2 = zext i16 %s2.elt1 to i32
1074  %mul1 = mul i32 %conv2, %conv
1075
1076  %s1.elt2 = extractelement <4 x i16> %vec1, i64 1
1077  %conv3 = zext i16 %s1.elt2 to i32
1078  %s2.elt2 = extractelement <4 x i16> %vec2, i64 1
1079  %conv4 = zext i16 %s2.elt2 to i32
1080  %mul2 = mul i32 %conv4, %conv3
1081
1082  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1083  %add = add i32 %mul2, %s3
1084  %add6 = add i32 %add, %mul1
1085  store i32 %add6, i32 addrspace(1)* %dst, align 4
1086  ret void
1087}
1088
1089define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1,
1090; GFX7-LABEL: udot2_v4i16_Hi:
1091; GFX7:       ; %bb.0: ; %entry
1092; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1093; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1094; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1095; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1096; GFX7-NEXT:    s_mov_b32 s2, -1
1097; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1098; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x1
1099; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x1
1100; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1101; GFX7-NEXT:    s_and_b32 s6, s4, s8
1102; GFX7-NEXT:    s_and_b32 s7, s5, s8
1103; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
1104; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
1105; GFX7-NEXT:    s_lshr_b32 s5, s5, 16
1106; GFX7-NEXT:    v_mov_b32_e32 v0, s4
1107; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1108; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1109; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1110; GFX7-NEXT:    v_mov_b32_e32 v1, s6
1111; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
1112; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1113; GFX7-NEXT:    s_endpgm
1114;
1115; GFX8-LABEL: udot2_v4i16_Hi:
1116; GFX8:       ; %bb.0: ; %entry
1117; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1118; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1119; GFX8-NEXT:    s_mov_b32 s2, 0xffff
1120; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1121; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x4
1122; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x4
1123; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
1124; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1125; GFX8-NEXT:    s_and_b32 s6, s3, s2
1126; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
1127; GFX8-NEXT:    s_and_b32 s2, s4, s2
1128; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
1129; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1130; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1131; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1132; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1133; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
1134; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1135; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1136; GFX8-NEXT:    flat_store_dword v[0:1], v2
1137; GFX8-NEXT:    s_endpgm
1138;
1139; GFX9-NODL-LABEL: udot2_v4i16_Hi:
1140; GFX9-NODL:       ; %bb.0: ; %entry
1141; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1142; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1143; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1144; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1145; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x4
1146; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x4
1147; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1148; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1149; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1150; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
1151; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1152; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
1153; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
1154; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1155; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1156; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1157; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
1158; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1159; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1160; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1161; GFX9-NODL-NEXT:    s_endpgm
1162;
1163; GFX9-DL-LABEL: udot2_v4i16_Hi:
1164; GFX9-DL:       ; %bb.0: ; %entry
1165; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1166; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1167; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1168; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x4
1169; GFX9-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
1170; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x4
1171; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1172; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s2
1173; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1174; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s4, v0, v1
1175; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1176; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1177; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1178; GFX9-DL-NEXT:    s_endpgm
1179;
1180; GFX10-DL-LABEL: udot2_v4i16_Hi:
1181; GFX10-DL:       ; %bb.0: ; %entry
1182; GFX10-DL-NEXT:    s_clause 0x1
1183; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1184; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1185; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1186; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1187; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
1188; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x4
1189; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x4
1190; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1191; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1192; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s1, s0, v0
1193; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
1194; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
1195; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1196; GFX10-DL-NEXT:    s_endpgm
1197                                          <4 x i16> addrspace(1)* %src2,
1198                                          i32 addrspace(1)* nocapture %dst) {
1199entry:
1200  %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
1201  %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
1202
1203  %s1.elt1 = extractelement <4 x i16> %vec1, i64 2
1204  %conv = zext i16 %s1.elt1 to i32
1205  %s2.elt1 = extractelement <4 x i16> %vec2, i64 2
1206  %conv2 = zext i16 %s2.elt1 to i32
1207  %mul1 = mul i32 %conv2, %conv
1208
1209  %s1.elt2 = extractelement <4 x i16> %vec1, i64 3
1210  %conv3 = zext i16 %s1.elt2 to i32
1211  %s2.elt2 = extractelement <4 x i16> %vec2, i64 3
1212  %conv4 = zext i16 %s2.elt2 to i32
1213  %mul2 = mul i32 %conv4, %conv3
1214
1215  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1216  %add = add i32 %mul2, %s3
1217  %add6 = add i32 %add, %mul1
1218  store i32 %add6, i32 addrspace(1)* %dst, align 4
1219  ret void
1220}
1221
1222define amdgpu_kernel void @notudot2_v4i16_Even(<4 x i16> addrspace(1)* %src1,
1223; GFX7-LABEL: notudot2_v4i16_Even:
1224; GFX7:       ; %bb.0: ; %entry
1225; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1226; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1227; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1228; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1229; GFX7-NEXT:    s_mov_b32 s2, -1
1230; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1231; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
1232; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
1233; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1234; GFX7-NEXT:    s_and_b32 s5, s5, s8
1235; GFX7-NEXT:    s_and_b32 s4, s4, s8
1236; GFX7-NEXT:    s_and_b32 s6, s6, s8
1237; GFX7-NEXT:    s_and_b32 s7, s7, s8
1238; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
1239; GFX7-NEXT:    v_mov_b32_e32 v0, s5
1240; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1241; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1242; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
1243; GFX7-NEXT:    v_mov_b32_e32 v1, s4
1244; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
1245; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1246; GFX7-NEXT:    s_endpgm
1247;
1248; GFX8-LABEL: notudot2_v4i16_Even:
1249; GFX8:       ; %bb.0: ; %entry
1250; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1251; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1252; GFX8-NEXT:    s_mov_b32 s8, 0xffff
1253; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1254; GFX8-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1255; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1256; GFX8-NEXT:    s_load_dword s6, s[0:1], 0x0
1257; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1258; GFX8-NEXT:    s_and_b32 s3, s3, s8
1259; GFX8-NEXT:    s_and_b32 s2, s2, s8
1260; GFX8-NEXT:    s_and_b32 s5, s5, s8
1261; GFX8-NEXT:    v_mov_b32_e32 v0, s6
1262; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1263; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1264; GFX8-NEXT:    s_and_b32 s4, s4, s8
1265; GFX8-NEXT:    v_mov_b32_e32 v1, s2
1266; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1267; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1268; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1269; GFX8-NEXT:    flat_store_dword v[0:1], v2
1270; GFX8-NEXT:    s_endpgm
1271;
1272; GFX9-NODL-LABEL: notudot2_v4i16_Even:
1273; GFX9-NODL:       ; %bb.0: ; %entry
1274; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1275; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1276; GFX9-NODL-NEXT:    s_mov_b32 s8, 0xffff
1277; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1278; GFX9-NODL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1279; GFX9-NODL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1280; GFX9-NODL-NEXT:    s_load_dword s6, s[0:1], 0x0
1281; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1282; GFX9-NODL-NEXT:    s_and_b32 s3, s3, s8
1283; GFX9-NODL-NEXT:    s_and_b32 s2, s2, s8
1284; GFX9-NODL-NEXT:    s_and_b32 s5, s5, s8
1285; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s6
1286; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1287; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1288; GFX9-NODL-NEXT:    s_and_b32 s4, s4, s8
1289; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
1290; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1291; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1292; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1293; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1294; GFX9-NODL-NEXT:    s_endpgm
1295;
1296; GFX9-DL-LABEL: notudot2_v4i16_Even:
1297; GFX9-DL:       ; %bb.0: ; %entry
1298; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1299; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1300; GFX9-DL-NEXT:    s_mov_b32 s8, 0xffff
1301; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1302; GFX9-DL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1303; GFX9-DL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1304; GFX9-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
1305; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1306; GFX9-DL-NEXT:    s_and_b32 s3, s3, s8
1307; GFX9-DL-NEXT:    s_and_b32 s2, s2, s8
1308; GFX9-DL-NEXT:    s_and_b32 s5, s5, s8
1309; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
1310; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1311; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1312; GFX9-DL-NEXT:    s_and_b32 s4, s4, s8
1313; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
1314; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1315; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1316; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1317; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1318; GFX9-DL-NEXT:    s_endpgm
1319;
1320; GFX10-DL-LABEL: notudot2_v4i16_Even:
1321; GFX10-DL:       ; %bb.0: ; %entry
1322; GFX10-DL-NEXT:    s_clause 0x1
1323; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1324; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1325; GFX10-DL-NEXT:    s_mov_b32 s7, 0xffff
1326; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1327; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1328; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
1329; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1330; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1331; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1332; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1333; GFX10-DL-NEXT:    s_and_b32 s1, s1, s7
1334; GFX10-DL-NEXT:    s_and_b32 s3, s3, s7
1335; GFX10-DL-NEXT:    s_and_b32 s0, s0, s7
1336; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s3, s1, v0
1337; GFX10-DL-NEXT:    s_and_b32 s1, s2, s7
1338; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v0
1339; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
1340; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
1341; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1342; GFX10-DL-NEXT:    s_endpgm
1343                                               <4 x i16> addrspace(1)* %src2,
1344                                               i32 addrspace(1)* nocapture %dst) {
1345entry:
1346  %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
1347  %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
1348
1349  %s1.elt1 = extractelement <4 x i16> %vec1, i64 0
1350  %conv = zext i16 %s1.elt1 to i32
1351  %s2.elt1 = extractelement <4 x i16> %vec2, i64 0
1352  %conv2 = zext i16 %s2.elt1 to i32
1353  %mul1 = mul i32 %conv2, %conv
1354
1355  %s1.elt2 = extractelement <4 x i16> %vec1, i64 2
1356  %conv3 = zext i16 %s1.elt2 to i32
1357  %s2.elt2 = extractelement <4 x i16> %vec2, i64 2
1358  %conv4 = zext i16 %s2.elt2 to i32
1359  %mul2 = mul i32 %conv4, %conv3
1360
1361  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1362  %add = add i32 %mul2, %s3
1363  %add6 = add i32 %add, %mul1
1364  store i32 %add6, i32 addrspace(1)* %dst, align 4
1365  ret void
1366}
1367
1368define amdgpu_kernel void @notudot2_v4i16_Middle(<4 x i16> addrspace(1)* %src1,
1369; GFX7-LABEL: notudot2_v4i16_Middle:
1370; GFX7:       ; %bb.0: ; %entry
1371; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1372; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1373; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1374; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1375; GFX7-NEXT:    s_mov_b32 s2, -1
1376; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1377; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
1378; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
1379; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1380; GFX7-NEXT:    s_and_b32 s5, s5, s8
1381; GFX7-NEXT:    s_and_b32 s7, s7, s8
1382; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
1383; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
1384; GFX7-NEXT:    v_mov_b32_e32 v0, s5
1385; GFX7-NEXT:    s_lshr_b32 s6, s6, 16
1386; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1387; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1388; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
1389; GFX7-NEXT:    v_mov_b32_e32 v1, s4
1390; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
1391; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1392; GFX7-NEXT:    s_endpgm
1393;
1394; GFX8-LABEL: notudot2_v4i16_Middle:
1395; GFX8:       ; %bb.0: ; %entry
1396; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1397; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1398; GFX8-NEXT:    s_mov_b32 s8, 0xffff
1399; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1400; GFX8-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1401; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1402; GFX8-NEXT:    s_load_dword s6, s[0:1], 0x0
1403; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1404; GFX8-NEXT:    s_and_b32 s3, s3, s8
1405; GFX8-NEXT:    s_lshr_b32 s2, s2, 16
1406; GFX8-NEXT:    s_and_b32 s5, s5, s8
1407; GFX8-NEXT:    v_mov_b32_e32 v0, s6
1408; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1409; GFX8-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1410; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
1411; GFX8-NEXT:    v_mov_b32_e32 v1, s2
1412; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1413; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1414; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1415; GFX8-NEXT:    flat_store_dword v[0:1], v2
1416; GFX8-NEXT:    s_endpgm
1417;
1418; GFX9-NODL-LABEL: notudot2_v4i16_Middle:
1419; GFX9-NODL:       ; %bb.0: ; %entry
1420; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1421; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1422; GFX9-NODL-NEXT:    s_mov_b32 s8, 0xffff
1423; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1424; GFX9-NODL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1425; GFX9-NODL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1426; GFX9-NODL-NEXT:    s_load_dword s6, s[0:1], 0x0
1427; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1428; GFX9-NODL-NEXT:    s_and_b32 s3, s3, s8
1429; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 16
1430; GFX9-NODL-NEXT:    s_and_b32 s5, s5, s8
1431; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s6
1432; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1433; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1434; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
1435; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
1436; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1437; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1438; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1439; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1440; GFX9-NODL-NEXT:    s_endpgm
1441;
1442; GFX9-DL-LABEL: notudot2_v4i16_Middle:
1443; GFX9-DL:       ; %bb.0: ; %entry
1444; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1445; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1446; GFX9-DL-NEXT:    s_mov_b32 s8, 0xffff
1447; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1448; GFX9-DL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1449; GFX9-DL-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
1450; GFX9-DL-NEXT:    s_load_dword s6, s[0:1], 0x0
1451; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1452; GFX9-DL-NEXT:    s_and_b32 s3, s3, s8
1453; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 16
1454; GFX9-DL-NEXT:    s_and_b32 s5, s5, s8
1455; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s6
1456; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1457; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
1458; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
1459; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
1460; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v1, v0
1461; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1462; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1463; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1464; GFX9-DL-NEXT:    s_endpgm
1465;
1466; GFX10-DL-LABEL: notudot2_v4i16_Middle:
1467; GFX10-DL:       ; %bb.0: ; %entry
1468; GFX10-DL-NEXT:    s_clause 0x1
1469; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1470; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1471; GFX10-DL-NEXT:    s_mov_b32 s7, 0xffff
1472; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1473; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1474; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
1475; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1476; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
1477; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1478; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1479; GFX10-DL-NEXT:    s_and_b32 s1, s1, s7
1480; GFX10-DL-NEXT:    s_and_b32 s3, s3, s7
1481; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 16
1482; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s3, s1, v0
1483; GFX10-DL-NEXT:    s_lshr_b32 s1, s2, 16
1484; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v0
1485; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
1486; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
1487; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1488; GFX10-DL-NEXT:    s_endpgm
1489                                                 <4 x i16> addrspace(1)* %src2,
1490                                                 i32 addrspace(1)* nocapture %dst) {
1491entry:
1492  %vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1
1493  %vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2
1494
1495  %s1.elt1 = extractelement <4 x i16> %vec1, i64 1
1496  %conv = zext i16 %s1.elt1 to i32
1497  %s2.elt1 = extractelement <4 x i16> %vec2, i64 1
1498  %conv2 = zext i16 %s2.elt1 to i32
1499  %mul1 = mul i32 %conv2, %conv
1500
1501  %s1.elt2 = extractelement <4 x i16> %vec1, i64 2
1502  %conv3 = zext i16 %s1.elt2 to i32
1503  %s2.elt2 = extractelement <4 x i16> %vec2, i64 2
1504  %conv4 = zext i16 %s2.elt2 to i32
1505  %mul2 = mul i32 %conv4, %conv3
1506
1507  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1508  %add = add i32 %mul2, %s3
1509  %add6 = add i32 %add, %mul1
1510  store i32 %add6, i32 addrspace(1)* %dst, align 4
1511  ret void
1512}
1513
1514define amdgpu_kernel void @notudot2_DiffIndex(<2 x i16> addrspace(1)* %src1,
1515; GFX7-LABEL: notudot2_DiffIndex:
1516; GFX7:       ; %bb.0: ; %entry
1517; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1518; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1519; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1520; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1521; GFX7-NEXT:    s_mov_b32 s2, -1
1522; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1523; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1524; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1525; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1526; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
1527; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
1528; GFX7-NEXT:    s_and_b32 s4, s4, s8
1529; GFX7-NEXT:    s_and_b32 s5, s5, s8
1530; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
1531; GFX7-NEXT:    v_mov_b32_e32 v0, s6
1532; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1533; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1534; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1535; GFX7-NEXT:    v_mov_b32_e32 v1, s4
1536; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
1537; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1538; GFX7-NEXT:    s_endpgm
1539;
1540; GFX8-LABEL: notudot2_DiffIndex:
1541; GFX8:       ; %bb.0: ; %entry
1542; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1543; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1544; GFX8-NEXT:    s_mov_b32 s2, 0xffff
1545; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1546; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1547; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1548; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
1549; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1550; GFX8-NEXT:    s_and_b32 s6, s3, s2
1551; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
1552; GFX8-NEXT:    s_and_b32 s2, s4, s2
1553; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1554; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1555; GFX8-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1556; GFX8-NEXT:    s_lshr_b32 s7, s4, 16
1557; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1558; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v1, v0
1559; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1560; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1561; GFX8-NEXT:    flat_store_dword v[0:1], v2
1562; GFX8-NEXT:    s_endpgm
1563;
1564; GFX9-NODL-LABEL: notudot2_DiffIndex:
1565; GFX9-NODL:       ; %bb.0: ; %entry
1566; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1567; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1568; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1569; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1570; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1571; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1572; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1573; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1574; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1575; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
1576; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1577; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
1578; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1579; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1580; GFX9-NODL-NEXT:    s_lshr_b32 s7, s4, 16
1581; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1582; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s7, v1, v0
1583; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1584; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1585; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1586; GFX9-NODL-NEXT:    s_endpgm
1587;
1588; GFX9-DL-LABEL: notudot2_DiffIndex:
1589; GFX9-DL:       ; %bb.0: ; %entry
1590; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1591; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1592; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
1593; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1594; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1595; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1596; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1597; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1598; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
1599; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
1600; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
1601; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
1602; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1603; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
1604; GFX9-DL-NEXT:    s_lshr_b32 s7, s4, 16
1605; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
1606; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s7, v1, v0
1607; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1608; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1609; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1610; GFX9-DL-NEXT:    s_endpgm
1611;
1612; GFX10-DL-LABEL: notudot2_DiffIndex:
1613; GFX10-DL:       ; %bb.0: ; %entry
1614; GFX10-DL-NEXT:    s_clause 0x1
1615; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1616; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1617; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1618; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1619; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
1620; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
1621; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
1622; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
1623; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1624; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1625; GFX10-DL-NEXT:    s_lshr_b32 s3, s0, 16
1626; GFX10-DL-NEXT:    s_and_b32 s6, s1, s2
1627; GFX10-DL-NEXT:    s_and_b32 s0, s0, s2
1628; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 16
1629; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s6, s3, v0
1630; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v0
1631; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
1632; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
1633; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1634; GFX10-DL-NEXT:    s_endpgm
1635                                              <2 x i16> addrspace(1)* %src2,
1636                                              i32 addrspace(1)* nocapture %dst) {
1637entry:
1638  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
1639  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
1640
1641  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
1642  %conv = zext i16 %s1.elt1 to i32
1643  %s2.elt1 = extractelement <2 x i16> %vec2, i64 1
1644  %conv2 = zext i16 %s2.elt1 to i32
1645  %mul1 = mul i32 %conv2, %conv
1646
1647  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
1648  %conv3 = zext i16 %s1.elt2 to i32
1649  %s2.elt2 = extractelement <2 x i16> %vec2, i64 0
1650  %conv4 = zext i16 %s2.elt2 to i32
1651  %mul2 = mul i32 %conv4, %conv3
1652
1653  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1654  %add = add i32 %mul2, %s3
1655  %add6 = add i32 %add, %mul1
1656  store i32 %add6, i32 addrspace(1)* %dst, align 4
1657  ret void
1658}
1659
1660define amdgpu_kernel void @udot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1,
1661; GFX7-LABEL: udot2_MultipleUses_add1:
1662; GFX7:       ; %bb.0: ; %entry
1663; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1664; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1665; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1666; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1667; GFX7-NEXT:    s_mov_b32 s2, -1
1668; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1669; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1670; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1671; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1672; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
1673; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
1674; GFX7-NEXT:    s_and_b32 s4, s4, s8
1675; GFX7-NEXT:    s_and_b32 s5, s5, s8
1676; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
1677; GFX7-NEXT:    v_mov_b32_e32 v0, s6
1678; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1679; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1680; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
1681; GFX7-NEXT:    v_mov_b32_e32 v1, s4
1682; GFX7-NEXT:    v_mad_u32_u24 v1, s5, v1, v0
1683; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
1684; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1685; GFX7-NEXT:    s_endpgm
1686;
1687; GFX8-LABEL: udot2_MultipleUses_add1:
1688; GFX8:       ; %bb.0: ; %entry
1689; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1690; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1691; GFX8-NEXT:    s_mov_b32 s2, 0xffff
1692; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1693; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1694; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1695; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
1696; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1697; GFX8-NEXT:    s_and_b32 s6, s3, s2
1698; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
1699; GFX8-NEXT:    s_and_b32 s2, s4, s2
1700; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
1701; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1702; GFX8-NEXT:    v_mov_b32_e32 v1, s3
1703; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1704; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1705; GFX8-NEXT:    v_mad_u32_u24 v1, s2, v1, v0
1706; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v1
1707; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1708; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1709; GFX8-NEXT:    flat_store_dword v[0:1], v2
1710; GFX8-NEXT:    s_endpgm
1711;
1712; GFX9-NODL-LABEL: udot2_MultipleUses_add1:
1713; GFX9-NODL:       ; %bb.0: ; %entry
1714; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1715; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1716; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
1717; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1718; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
1719; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
1720; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
1721; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1722; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
1723; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
1724; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
1725; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
1726; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
1727; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
1728; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1729; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
1730; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, s2, v1, v0
1731; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v1, v0
1732; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1733; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1734; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1735; GFX9-NODL-NEXT:    s_endpgm
1736;
1737; GFX9-DL-LABEL: udot2_MultipleUses_add1:
1738; GFX9-DL:       ; %bb.0: ; %entry
1739; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1740; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1741; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
1742; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1743; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
1744; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
1745; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
1746; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1747; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
1748; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
1749; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
1750; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
1751; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
1752; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
1753; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
1754; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
1755; GFX9-DL-NEXT:    v_mad_u32_u24 v1, s2, v1, v0
1756; GFX9-DL-NEXT:    v_add_u32_e32 v2, v1, v0
1757; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1758; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1759; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1760; GFX9-DL-NEXT:    s_endpgm
1761;
1762; GFX10-DL-LABEL: udot2_MultipleUses_add1:
1763; GFX10-DL:       ; %bb.0: ; %entry
1764; GFX10-DL-NEXT:    s_clause 0x1
1765; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1766; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1767; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1768; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1769; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
1770; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
1771; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
1772; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1773; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1774; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 16
1775; GFX10-DL-NEXT:    s_lshr_b32 s3, s1, 16
1776; GFX10-DL-NEXT:    s_mov_b32 s6, 0xffff
1777; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s3, s2, v0
1778; GFX10-DL-NEXT:    s_and_b32 s0, s0, s6
1779; GFX10-DL-NEXT:    s_and_b32 s1, s1, s6
1780; GFX10-DL-NEXT:    v_mad_u32_u24 v1, s1, s0, v0
1781; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v1, v0
1782; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
1783; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
1784; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1785; GFX10-DL-NEXT:    s_endpgm
1786                                                   <2 x i16> addrspace(1)* %src2,
1787                                                   i32 addrspace(1)* nocapture %dst) {
1788entry:
1789  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
1790  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
1791
1792  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
1793  %conv = zext i16 %s1.elt1 to i32
1794  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
1795  %conv2 = zext i16 %s2.elt1 to i32
1796  %mul1 = mul i32 %conv2, %conv
1797
1798  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
1799  %conv3 = zext i16 %s1.elt2 to i32
1800  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
1801  %conv4 = zext i16 %s2.elt2 to i32
1802  %mul2 = mul i32 %conv4, %conv3
1803
1804  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1805  %add1 = add i32 %mul2, %s3
1806  %add2 = add i32 %add1, %mul1
1807
1808  %res = add i32 %add2, %add1
1809  store i32 %res, i32 addrspace(1)* %dst, align 4
1810  ret void
1811}
1812
1813define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1,
1814; GFX7-LABEL: idot2_MultipleUses_add1:
1815; GFX7:       ; %bb.0: ; %entry
1816; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1817; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1818; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1819; GFX7-NEXT:    s_mov_b32 s2, -1
1820; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1821; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1822; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1823; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
1824; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1825; GFX7-NEXT:    s_sext_i32_i16 s7, s4
1826; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
1827; GFX7-NEXT:    s_sext_i32_i16 s8, s5
1828; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
1829; GFX7-NEXT:    v_mov_b32_e32 v0, s4
1830; GFX7-NEXT:    v_mov_b32_e32 v1, s6
1831; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
1832; GFX7-NEXT:    v_mov_b32_e32 v1, s7
1833; GFX7-NEXT:    v_mad_i32_i24 v1, s8, v1, v0
1834; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
1835; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1836; GFX7-NEXT:    s_endpgm
1837;
1838; GFX8-LABEL: idot2_MultipleUses_add1:
1839; GFX8:       ; %bb.0: ; %entry
1840; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1841; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1842; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1843; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
1844; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
1845; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
1846; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1847; GFX8-NEXT:    s_sext_i32_i16 s5, s2
1848; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
1849; GFX8-NEXT:    s_sext_i32_i16 s6, s3
1850; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
1851; GFX8-NEXT:    v_mov_b32_e32 v0, s4
1852; GFX8-NEXT:    v_mov_b32_e32 v1, s2
1853; GFX8-NEXT:    v_mov_b32_e32 v2, s5
1854; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
1855; GFX8-NEXT:    v_mad_i32_i24 v1, s6, v2, v0
1856; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v1
1857; GFX8-NEXT:    v_mov_b32_e32 v0, s0
1858; GFX8-NEXT:    v_mov_b32_e32 v1, s1
1859; GFX8-NEXT:    flat_store_dword v[0:1], v2
1860; GFX8-NEXT:    s_endpgm
1861;
1862; GFX9-NODL-LABEL: idot2_MultipleUses_add1:
1863; GFX9-NODL:       ; %bb.0: ; %entry
1864; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1865; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1866; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1867; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
1868; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
1869; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
1870; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
1871; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
1872; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
1873; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
1874; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
1875; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
1876; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
1877; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
1878; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
1879; GFX9-NODL-NEXT:    v_mad_i32_i24 v1, s6, v2, v0
1880; GFX9-NODL-NEXT:    v_add_u32_e32 v2, v1, v0
1881; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
1882; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
1883; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
1884; GFX9-NODL-NEXT:    s_endpgm
1885;
1886; GFX9-DL-LABEL: idot2_MultipleUses_add1:
1887; GFX9-DL:       ; %bb.0: ; %entry
1888; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1889; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1890; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1891; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
1892; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
1893; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
1894; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
1895; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
1896; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
1897; GFX9-DL-NEXT:    s_sext_i32_i16 s6, s3
1898; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
1899; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
1900; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
1901; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
1902; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
1903; GFX9-DL-NEXT:    v_mad_i32_i24 v1, s6, v2, v0
1904; GFX9-DL-NEXT:    v_add_u32_e32 v2, v1, v0
1905; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
1906; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
1907; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
1908; GFX9-DL-NEXT:    s_endpgm
1909;
1910; GFX10-DL-LABEL: idot2_MultipleUses_add1:
1911; GFX10-DL:       ; %bb.0: ; %entry
1912; GFX10-DL-NEXT:    s_clause 0x1
1913; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
1914; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1915; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
1916; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1917; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
1918; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
1919; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
1920; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
1921; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
1922; GFX10-DL-NEXT:    s_ashr_i32 s2, s0, 16
1923; GFX10-DL-NEXT:    s_ashr_i32 s3, s1, 16
1924; GFX10-DL-NEXT:    s_sext_i32_i16 s0, s0
1925; GFX10-DL-NEXT:    s_sext_i32_i16 s1, s1
1926; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s3, s2, v0
1927; GFX10-DL-NEXT:    v_mad_i32_i24 v1, s1, s0, v0
1928; GFX10-DL-NEXT:    v_add_nc_u32_e32 v2, v1, v0
1929; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
1930; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
1931; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
1932; GFX10-DL-NEXT:    s_endpgm
1933                                                   <2 x i16> addrspace(1)* %src2,
1934                                                   i32 addrspace(1)* nocapture %dst) {
1935entry:
1936  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
1937  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
1938
1939  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
1940  %conv = sext i16 %s1.elt1 to i32
1941  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
1942  %conv2 = sext i16 %s2.elt1 to i32
1943  %mul1 = mul i32 %conv2, %conv
1944
1945  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
1946  %conv3 = sext i16 %s1.elt2 to i32
1947  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
1948  %conv4 = sext i16 %s2.elt2 to i32
1949  %mul2 = mul i32 %conv4, %conv3
1950
1951  %s3 = load i32, i32 addrspace(1)* %dst, align 4
1952  %add1 = add i32 %mul2, %s3
1953  %add2 = add i32 %add1, %mul1
1954
1955  %res = add i32 %add2, %add1
1956  store i32 %res, i32 addrspace(1)* %dst, align 4
1957  ret void
1958}
1959
1960define amdgpu_kernel void @udot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1,
1961; GFX7-LABEL: udot2_MultipleUses_mul1:
1962; GFX7:       ; %bb.0: ; %entry
1963; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
1964; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1965; GFX7-NEXT:    s_mov_b32 s8, 0xffff
1966; GFX7-NEXT:    s_mov_b32 s3, 0xf000
1967; GFX7-NEXT:    s_mov_b32 s2, -1
1968; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1969; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
1970; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
1971; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1972; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
1973; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
1974; GFX7-NEXT:    s_and_b32 s4, s4, s8
1975; GFX7-NEXT:    s_and_b32 s5, s5, s8
1976; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
1977; GFX7-NEXT:    v_mov_b32_e32 v0, s4
1978; GFX7-NEXT:    v_mov_b32_e32 v2, s6
1979; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1980; GFX7-NEXT:    v_mov_b32_e32 v1, s8
1981; GFX7-NEXT:    v_mad_u32_u24 v1, s5, v0, v1
1982; GFX7-NEXT:    v_mad_u32_u24 v1, s7, v2, v1
1983; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v0, v1
1984; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1985; GFX7-NEXT:    s_endpgm
1986;
1987; GFX8-LABEL: udot2_MultipleUses_mul1:
1988; GFX8:       ; %bb.0: ; %entry
1989; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1990; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1991; GFX8-NEXT:    s_mov_b32 s2, 0xffff
1992; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1993; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
1994; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
1995; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
1996; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1997; GFX8-NEXT:    s_and_b32 s6, s3, s2
1998; GFX8-NEXT:    s_and_b32 s2, s4, s2
1999; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
2000; GFX8-NEXT:    v_mov_b32_e32 v0, s5
2001; GFX8-NEXT:    v_mov_b32_e32 v1, s6
2002; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
2003; GFX8-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
2004; GFX8-NEXT:    v_mov_b32_e32 v2, s3
2005; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v2, v0
2006; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2007; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2008; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2009; GFX8-NEXT:    flat_store_dword v[0:1], v2
2010; GFX8-NEXT:    s_endpgm
2011;
2012; GFX9-NODL-LABEL: udot2_MultipleUses_mul1:
2013; GFX9-NODL:       ; %bb.0: ; %entry
2014; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2015; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2016; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
2017; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2018; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
2019; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
2020; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
2021; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2022; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
2023; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
2024; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
2025; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
2026; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
2027; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
2028; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
2029; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s3
2030; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v2, v0
2031; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2032; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2033; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2034; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2035; GFX9-NODL-NEXT:    s_endpgm
2036;
2037; GFX9-DL-LABEL: udot2_MultipleUses_mul1:
2038; GFX9-DL:       ; %bb.0: ; %entry
2039; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2040; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2041; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
2042; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2043; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
2044; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
2045; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
2046; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2047; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
2048; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
2049; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
2050; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
2051; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
2052; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
2053; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s2, v1, v0
2054; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
2055; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v2, v0
2056; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2057; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2058; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2059; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2060; GFX9-DL-NEXT:    s_endpgm
2061;
2062; GFX10-DL-LABEL: udot2_MultipleUses_mul1:
2063; GFX10-DL:       ; %bb.0: ; %entry
2064; GFX10-DL-NEXT:    s_clause 0x1
2065; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
2066; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2067; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2068; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2069; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
2070; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
2071; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
2072; GFX10-DL-NEXT:    s_mov_b32 s2, 0xffff
2073; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2074; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
2075; GFX10-DL-NEXT:    s_and_b32 s3, s0, s2
2076; GFX10-DL-NEXT:    s_and_b32 s2, s1, s2
2077; GFX10-DL-NEXT:    s_lshr_b32 s0, s0, 16
2078; GFX10-DL-NEXT:    s_lshr_b32 s1, s1, 16
2079; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s2, s3, v0
2080; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s1, s0, v0
2081; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s2, s3, v0
2082; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
2083; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
2084; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2085; GFX10-DL-NEXT:    s_endpgm
2086                                                   <2 x i16> addrspace(1)* %src2,
2087                                                   i32 addrspace(1)* nocapture %dst) {
2088entry:
2089  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2090  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2091
2092  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
2093  %conv = zext i16 %s1.elt1 to i32
2094  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
2095  %conv2 = zext i16 %s2.elt1 to i32
2096  %mul1 = mul i32 %conv2, %conv
2097
2098  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
2099  %conv3 = zext i16 %s1.elt2 to i32
2100  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
2101  %conv4 = zext i16 %s2.elt2 to i32
2102  %mul2 = mul i32 %conv4, %conv3
2103
2104  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2105  %add0 = add i32 %mul1, %s3
2106
2107  %add1 = add i32 %mul2, %add0
2108  %add2 = add i32 %add1, %mul1
2109
2110  store i32 %add2, i32 addrspace(1)* %dst, align 4
2111  ret void
2112}
2113
2114define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1,
2115; GFX7-LABEL: idot2_MultipleUses_mul1:
2116; GFX7:       ; %bb.0: ; %entry
2117; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2118; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2119; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2120; GFX7-NEXT:    s_mov_b32 s2, -1
2121; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2122; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2123; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2124; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
2125; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2126; GFX7-NEXT:    s_sext_i32_i16 s7, s4
2127; GFX7-NEXT:    s_sext_i32_i16 s8, s5
2128; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
2129; GFX7-NEXT:    v_mov_b32_e32 v0, s7
2130; GFX7-NEXT:    v_mov_b32_e32 v1, s6
2131; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
2132; GFX7-NEXT:    v_mad_i32_i24 v1, s8, v0, v1
2133; GFX7-NEXT:    v_mov_b32_e32 v2, s4
2134; GFX7-NEXT:    v_mad_i32_i24 v1, s5, v2, v1
2135; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v0, v1
2136; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2137; GFX7-NEXT:    s_endpgm
2138;
2139; GFX8-LABEL: idot2_MultipleUses_mul1:
2140; GFX8:       ; %bb.0: ; %entry
2141; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2142; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2143; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2144; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
2145; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
2146; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
2147; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2148; GFX8-NEXT:    s_sext_i32_i16 s5, s2
2149; GFX8-NEXT:    s_sext_i32_i16 s6, s3
2150; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
2151; GFX8-NEXT:    v_mov_b32_e32 v0, s4
2152; GFX8-NEXT:    v_mov_b32_e32 v1, s5
2153; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
2154; GFX8-NEXT:    v_mov_b32_e32 v2, s2
2155; GFX8-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
2156; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v2, v0
2157; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v1, v0
2158; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2159; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2160; GFX8-NEXT:    flat_store_dword v[0:1], v2
2161; GFX8-NEXT:    s_endpgm
2162;
2163; GFX9-NODL-LABEL: idot2_MultipleUses_mul1:
2164; GFX9-NODL:       ; %bb.0: ; %entry
2165; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2166; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2167; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2168; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
2169; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
2170; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
2171; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2172; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
2173; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
2174; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
2175; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
2176; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s5
2177; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
2178; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s2
2179; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
2180; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v2, v0
2181; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v1, v0
2182; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2183; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2184; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2185; GFX9-NODL-NEXT:    s_endpgm
2186;
2187; GFX9-DL-LABEL: idot2_MultipleUses_mul1:
2188; GFX9-DL:       ; %bb.0: ; %entry
2189; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2190; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2191; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2192; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2193; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2194; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
2195; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2196; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
2197; GFX9-DL-NEXT:    s_sext_i32_i16 s6, s3
2198; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
2199; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
2200; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s5
2201; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
2202; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s2
2203; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s6, v1, v0
2204; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v2, v0
2205; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v1, v0
2206; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2207; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2208; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2209; GFX9-DL-NEXT:    s_endpgm
2210;
2211; GFX10-DL-LABEL: idot2_MultipleUses_mul1:
2212; GFX10-DL:       ; %bb.0: ; %entry
2213; GFX10-DL-NEXT:    s_clause 0x1
2214; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
2215; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2216; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2217; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2218; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
2219; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
2220; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
2221; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2222; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
2223; GFX10-DL-NEXT:    s_sext_i32_i16 s2, s0
2224; GFX10-DL-NEXT:    s_sext_i32_i16 s3, s1
2225; GFX10-DL-NEXT:    s_ashr_i32 s0, s0, 16
2226; GFX10-DL-NEXT:    s_ashr_i32 s1, s1, 16
2227; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s3, s2, v0
2228; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s1, s0, v0
2229; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s3, s2, v0
2230; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
2231; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
2232; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2233; GFX10-DL-NEXT:    s_endpgm
2234                                                   <2 x i16> addrspace(1)* %src2,
2235                                                   i32 addrspace(1)* nocapture %dst) {
2236entry:
2237  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2238  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2239
2240  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
2241  %conv = sext i16 %s1.elt1 to i32
2242  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
2243  %conv2 = sext i16 %s2.elt1 to i32
2244  %mul1 = mul i32 %conv2, %conv
2245
2246  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
2247  %conv3 = sext i16 %s1.elt2 to i32
2248  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
2249  %conv4 = sext i16 %s2.elt2 to i32
2250  %mul2 = mul i32 %conv4, %conv3
2251
2252  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2253  %add0 = add i32 %mul1, %s3
2254
2255  %add1 = add i32 %mul2, %add0
2256  %add2 = add i32 %add1, %mul1
2257
2258  store i32 %add2, i32 addrspace(1)* %dst, align 4
2259  ret void
2260}
2261
2262define amdgpu_kernel void @udot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1,
2263; GFX7-LABEL: udot2_MultipleUses_mul2:
2264; GFX7:       ; %bb.0: ; %entry
2265; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2266; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2267; GFX7-NEXT:    s_mov_b32 s8, 0xffff
2268; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2269; GFX7-NEXT:    s_mov_b32 s2, -1
2270; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2271; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2272; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2273; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2274; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
2275; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
2276; GFX7-NEXT:    s_and_b32 s4, s4, s8
2277; GFX7-NEXT:    s_and_b32 s5, s5, s8
2278; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x0
2279; GFX7-NEXT:    v_mov_b32_e32 v0, s6
2280; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2281; GFX7-NEXT:    v_mov_b32_e32 v1, s8
2282; GFX7-NEXT:    v_mad_u32_u24 v1, s7, v0, v1
2283; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v0, v1
2284; GFX7-NEXT:    v_mov_b32_e32 v1, s4
2285; GFX7-NEXT:    v_mad_u32_u24 v0, s5, v1, v0
2286; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2287; GFX7-NEXT:    s_endpgm
2288;
2289; GFX8-LABEL: udot2_MultipleUses_mul2:
2290; GFX8:       ; %bb.0: ; %entry
2291; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2292; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2293; GFX8-NEXT:    s_mov_b32 s2, 0xffff
2294; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2295; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x0
2296; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
2297; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
2298; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2299; GFX8-NEXT:    s_and_b32 s6, s3, s2
2300; GFX8-NEXT:    s_lshr_b32 s3, s3, 16
2301; GFX8-NEXT:    s_and_b32 s2, s4, s2
2302; GFX8-NEXT:    s_lshr_b32 s4, s4, 16
2303; GFX8-NEXT:    v_mov_b32_e32 v0, s5
2304; GFX8-NEXT:    v_mov_b32_e32 v1, s3
2305; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2306; GFX8-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2307; GFX8-NEXT:    v_mov_b32_e32 v1, s6
2308; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2309; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2310; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2311; GFX8-NEXT:    flat_store_dword v[0:1], v2
2312; GFX8-NEXT:    s_endpgm
2313;
2314; GFX9-NODL-LABEL: udot2_MultipleUses_mul2:
2315; GFX9-NODL:       ; %bb.0: ; %entry
2316; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2317; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2318; GFX9-NODL-NEXT:    s_mov_b32 s2, 0xffff
2319; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2320; GFX9-NODL-NEXT:    s_load_dword s3, s[4:5], 0x0
2321; GFX9-NODL-NEXT:    s_load_dword s4, s[6:7], 0x0
2322; GFX9-NODL-NEXT:    s_load_dword s5, s[0:1], 0x0
2323; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2324; GFX9-NODL-NEXT:    s_and_b32 s6, s3, s2
2325; GFX9-NODL-NEXT:    s_lshr_b32 s3, s3, 16
2326; GFX9-NODL-NEXT:    s_and_b32 s2, s4, s2
2327; GFX9-NODL-NEXT:    s_lshr_b32 s4, s4, 16
2328; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s5
2329; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s3
2330; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2331; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2332; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s6
2333; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2334; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2335; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2336; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2337; GFX9-NODL-NEXT:    s_endpgm
2338;
2339; GFX9-DL-LABEL: udot2_MultipleUses_mul2:
2340; GFX9-DL:       ; %bb.0: ; %entry
2341; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2342; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2343; GFX9-DL-NEXT:    s_mov_b32 s2, 0xffff
2344; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2345; GFX9-DL-NEXT:    s_load_dword s3, s[4:5], 0x0
2346; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
2347; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
2348; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2349; GFX9-DL-NEXT:    s_and_b32 s6, s3, s2
2350; GFX9-DL-NEXT:    s_lshr_b32 s3, s3, 16
2351; GFX9-DL-NEXT:    s_and_b32 s2, s4, s2
2352; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 16
2353; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s5
2354; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s3
2355; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2356; GFX9-DL-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2357; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s6
2358; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s2, v1, v0
2359; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2360; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2361; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2362; GFX9-DL-NEXT:    s_endpgm
2363;
2364; GFX10-DL-LABEL: udot2_MultipleUses_mul2:
2365; GFX10-DL:       ; %bb.0: ; %entry
2366; GFX10-DL-NEXT:    s_clause 0x1
2367; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
2368; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2369; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2370; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2371; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
2372; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
2373; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
2374; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2375; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
2376; GFX10-DL-NEXT:    s_lshr_b32 s2, s0, 16
2377; GFX10-DL-NEXT:    s_lshr_b32 s3, s1, 16
2378; GFX10-DL-NEXT:    s_mov_b32 s6, 0xffff
2379; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s3, s2, v0
2380; GFX10-DL-NEXT:    s_and_b32 s0, s0, s6
2381; GFX10-DL-NEXT:    s_and_b32 s1, s1, s6
2382; GFX10-DL-NEXT:    v_mad_u32_u24 v0, s3, s2, v0
2383; GFX10-DL-NEXT:    v_mad_u32_u24 v2, s1, s0, v0
2384; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
2385; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
2386; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2387; GFX10-DL-NEXT:    s_endpgm
2388                                                   <2 x i16> addrspace(1)* %src2,
2389                                                   i32 addrspace(1)* nocapture %dst) {
2390entry:
2391  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2392  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2393
2394  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
2395  %conv = zext i16 %s1.elt1 to i32
2396  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
2397  %conv2 = zext i16 %s2.elt1 to i32
2398  %mul1 = mul i32 %conv2, %conv
2399
2400  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
2401  %conv3 = zext i16 %s1.elt2 to i32
2402  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
2403  %conv4 = zext i16 %s2.elt2 to i32
2404  %mul2 = mul i32 %conv4, %conv3
2405
2406  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2407  %add0 = add i32 %mul2, %s3
2408
2409  %add1 = add i32 %mul2, %add0
2410  %add2 = add i32 %add1, %mul1
2411
2412  store i32 %add2, i32 addrspace(1)* %dst, align 4
2413  ret void
2414}
2415
2416define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1,
2417; GFX7-LABEL: idot2_MultipleUses_mul2:
2418; GFX7:       ; %bb.0: ; %entry
2419; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2420; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2421; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2422; GFX7-NEXT:    s_mov_b32 s2, -1
2423; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2424; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2425; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2426; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
2427; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2428; GFX7-NEXT:    s_sext_i32_i16 s7, s4
2429; GFX7-NEXT:    s_ashr_i32 s4, s4, 16
2430; GFX7-NEXT:    s_sext_i32_i16 s8, s5
2431; GFX7-NEXT:    s_ashr_i32 s5, s5, 16
2432; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2433; GFX7-NEXT:    v_mov_b32_e32 v1, s6
2434; GFX7-NEXT:    v_mad_i32_i24 v1, s5, v0, v1
2435; GFX7-NEXT:    v_mad_i32_i24 v0, s5, v0, v1
2436; GFX7-NEXT:    v_mov_b32_e32 v1, s7
2437; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
2438; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2439; GFX7-NEXT:    s_endpgm
2440;
2441; GFX8-LABEL: idot2_MultipleUses_mul2:
2442; GFX8:       ; %bb.0: ; %entry
2443; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2444; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2445; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2446; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
2447; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
2448; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
2449; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2450; GFX8-NEXT:    s_sext_i32_i16 s5, s2
2451; GFX8-NEXT:    s_ashr_i32 s2, s2, 16
2452; GFX8-NEXT:    s_sext_i32_i16 s6, s3
2453; GFX8-NEXT:    s_ashr_i32 s3, s3, 16
2454; GFX8-NEXT:    v_mov_b32_e32 v0, s4
2455; GFX8-NEXT:    v_mov_b32_e32 v1, s2
2456; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
2457; GFX8-NEXT:    v_mov_b32_e32 v2, s5
2458; GFX8-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
2459; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
2460; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2461; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2462; GFX8-NEXT:    flat_store_dword v[0:1], v2
2463; GFX8-NEXT:    s_endpgm
2464;
2465; GFX9-NODL-LABEL: idot2_MultipleUses_mul2:
2466; GFX9-NODL:       ; %bb.0: ; %entry
2467; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2468; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2469; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2470; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
2471; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
2472; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
2473; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2474; GFX9-NODL-NEXT:    s_sext_i32_i16 s5, s2
2475; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 16
2476; GFX9-NODL-NEXT:    s_sext_i32_i16 s6, s3
2477; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 16
2478; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
2479; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s2
2480; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
2481; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s5
2482; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
2483; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
2484; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2485; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2486; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2487; GFX9-NODL-NEXT:    s_endpgm
2488;
2489; GFX9-DL-LABEL: idot2_MultipleUses_mul2:
2490; GFX9-DL:       ; %bb.0: ; %entry
2491; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2492; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2493; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2494; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2495; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2496; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
2497; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2498; GFX9-DL-NEXT:    s_sext_i32_i16 s5, s2
2499; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 16
2500; GFX9-DL-NEXT:    s_sext_i32_i16 s6, s3
2501; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 16
2502; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
2503; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s2
2504; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
2505; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s5
2506; GFX9-DL-NEXT:    v_mad_i32_i24 v0, s3, v1, v0
2507; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v2, v0
2508; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2509; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2510; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2511; GFX9-DL-NEXT:    s_endpgm
2512;
2513; GFX10-DL-LABEL: idot2_MultipleUses_mul2:
2514; GFX10-DL:       ; %bb.0: ; %entry
2515; GFX10-DL-NEXT:    s_clause 0x1
2516; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
2517; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2518; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2519; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2520; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
2521; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
2522; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
2523; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2524; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s6
2525; GFX10-DL-NEXT:    s_ashr_i32 s2, s0, 16
2526; GFX10-DL-NEXT:    s_ashr_i32 s3, s1, 16
2527; GFX10-DL-NEXT:    s_sext_i32_i16 s0, s0
2528; GFX10-DL-NEXT:    s_sext_i32_i16 s1, s1
2529; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s3, s2, v0
2530; GFX10-DL-NEXT:    v_mad_i32_i24 v0, s3, s2, v0
2531; GFX10-DL-NEXT:    v_mad_i32_i24 v2, s1, s0, v0
2532; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
2533; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
2534; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2535; GFX10-DL-NEXT:    s_endpgm
2536                                                   <2 x i16> addrspace(1)* %src2,
2537                                                   i32 addrspace(1)* nocapture %dst) {
2538entry:
2539  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2540  %vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2541
2542  %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
2543  %conv = sext i16 %s1.elt1 to i32
2544  %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
2545  %conv2 = sext i16 %s2.elt1 to i32
2546  %mul1 = mul i32 %conv2, %conv
2547
2548  %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
2549  %conv3 = sext i16 %s1.elt2 to i32
2550  %s2.elt2 = extractelement <2 x i16> %vec2, i64 1
2551  %conv4 = sext i16 %s2.elt2 to i32
2552  %mul2 = mul i32 %conv4, %conv3
2553
2554  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2555  %add0 = add i32 %mul2, %s3
2556
2557  %add1 = add i32 %mul2, %add0
2558  %add2 = add i32 %add1, %mul1
2559
2560  store i32 %add2, i32 addrspace(1)* %dst, align 4
2561  ret void
2562}
2563
2564define amdgpu_kernel void @udot2_acc16(<2 x i16> addrspace(1)* %src1,
2565; GFX7-LABEL: udot2_acc16:
2566; GFX7:       ; %bb.0: ; %entry
2567; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2568; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
2569; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2570; GFX7-NEXT:    s_mov_b32 s2, -1
2571; GFX7-NEXT:    s_mov_b32 s8, 0xffff
2572; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2573; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
2574; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
2575; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
2576; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2577; GFX7-NEXT:    s_lshr_b32 s6, s4, 16
2578; GFX7-NEXT:    s_lshr_b32 s7, s5, 16
2579; GFX7-NEXT:    v_mov_b32_e32 v1, s7
2580; GFX7-NEXT:    s_and_b32 s5, s5, s8
2581; GFX7-NEXT:    s_and_b32 s4, s4, s8
2582; GFX7-NEXT:    s_waitcnt vmcnt(0)
2583; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
2584; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2585; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
2586; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
2587; GFX7-NEXT:    s_endpgm
2588;
2589; GFX8-LABEL: udot2_acc16:
2590; GFX8:       ; %bb.0: ; %entry
2591; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2592; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2593; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2594; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
2595; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2596; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2597; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
2598; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
2599; GFX8-NEXT:    s_mov_b32 s0, 0xffff
2600; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2601; GFX8-NEXT:    s_and_b32 s3, s1, s0
2602; GFX8-NEXT:    s_lshr_b32 s1, s1, 16
2603; GFX8-NEXT:    s_and_b32 s0, s2, s0
2604; GFX8-NEXT:    s_lshr_b32 s2, s2, 16
2605; GFX8-NEXT:    v_mov_b32_e32 v3, s1
2606; GFX8-NEXT:    s_waitcnt vmcnt(0)
2607; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
2608; GFX8-NEXT:    v_mov_b32_e32 v3, s3
2609; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
2610; GFX8-NEXT:    flat_store_short v[0:1], v2
2611; GFX8-NEXT:    s_endpgm
2612;
2613; GFX9-NODL-LABEL: udot2_acc16:
2614; GFX9-NODL:       ; %bb.0: ; %entry
2615; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2616; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2617; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2618; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
2619; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2620; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2621; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
2622; GFX9-NODL-NEXT:    s_load_dword s1, s[6:7], 0x0
2623; GFX9-NODL-NEXT:    s_mov_b32 s0, 0xffff
2624; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2625; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s0
2626; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 16
2627; GFX9-NODL-NEXT:    s_and_b32 s0, s2, s0
2628; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 16
2629; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
2630; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
2631; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
2632; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
2633; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
2634; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
2635; GFX9-NODL-NEXT:    s_endpgm
2636;
2637; GFX9-DL-LABEL: udot2_acc16:
2638; GFX9-DL:       ; %bb.0: ; %entry
2639; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2640; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2641; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2642; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
2643; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
2644; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2645; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2646; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
2647; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2648; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
2649; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2650; GFX9-DL-NEXT:    v_dot2_u32_u16 v2, s2, v3, v2
2651; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
2652; GFX9-DL-NEXT:    s_endpgm
2653;
2654; GFX10-DL-LABEL: udot2_acc16:
2655; GFX10-DL:       ; %bb.0: ; %entry
2656; GFX10-DL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
2657; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2658; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2659; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s2
2660; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s3
2661; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2662; GFX10-DL-NEXT:    global_load_ushort v2, v[0:1], off
2663; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2664; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
2665; GFX10-DL-NEXT:    s_load_dword s1, s[2:3], 0x0
2666; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2667; GFX10-DL-NEXT:    v_dot2_u32_u16 v2, s0, s1, v2
2668; GFX10-DL-NEXT:    global_store_short v[0:1], v2, off
2669; GFX10-DL-NEXT:    s_endpgm
2670                                       <2 x i16> addrspace(1)* %src2,
2671                                       i16 addrspace(1)* nocapture %dst) {
2672entry:
2673  %v1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1
2674  %v2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2
2675
2676  %v1e1 = extractelement <2 x i16> %v1, i64 0
2677  %v2e1 = extractelement <2 x i16> %v2, i64 0
2678  %mul1 = mul i16 %v1e1, %v2e1
2679
2680  %v1e2 = extractelement <2 x i16> %v1, i64 1
2681  %v2e2 = extractelement <2 x i16> %v2, i64 1
2682  %mul2 = mul i16 %v1e2, %v2e2
2683
2684  %s2 = load i16, i16 addrspace(1)* %dst, align 2
2685  %add1 = add i16 %mul2, %s2
2686  %add2 = add i16 %add1, %mul1
2687  store i16 %add2, i16 addrspace(1)* %dst, align 2
2688  ret void
2689}
2690
2691define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
2692; GFX7-LABEL: notsdot2_sext8:
2693; GFX7:       ; %bb.0: ; %entry
2694; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
2695; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
2696; GFX7-NEXT:    s_mov_b32 s3, 0xf000
2697; GFX7-NEXT:    s_mov_b32 s2, -1
2698; GFX7-NEXT:    s_mov_b32 s10, s2
2699; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2700; GFX7-NEXT:    s_mov_b32 s0, s4
2701; GFX7-NEXT:    s_mov_b32 s1, s5
2702; GFX7-NEXT:    s_mov_b32 s4, s6
2703; GFX7-NEXT:    s_mov_b32 s5, s7
2704; GFX7-NEXT:    s_mov_b32 s6, s2
2705; GFX7-NEXT:    s_mov_b32 s7, s3
2706; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
2707; GFX7-NEXT:    buffer_load_ushort v1, off, s[4:7], 0
2708; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
2709; GFX7-NEXT:    s_mov_b32 s11, s3
2710; GFX7-NEXT:    s_waitcnt vmcnt(1)
2711; GFX7-NEXT:    v_bfe_i32 v2, v0, 0, 8
2712; GFX7-NEXT:    s_waitcnt vmcnt(0)
2713; GFX7-NEXT:    v_bfe_i32 v3, v1, 0, 8
2714; GFX7-NEXT:    v_bfe_i32 v0, v0, 8, 8
2715; GFX7-NEXT:    v_bfe_i32 v1, v1, 8, 8
2716; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2717; GFX7-NEXT:    v_mad_i32_i24 v0, v1, v0, s0
2718; GFX7-NEXT:    v_mad_i32_i24 v0, v3, v2, v0
2719; GFX7-NEXT:    buffer_store_dword v0, off, s[8:11], 0
2720; GFX7-NEXT:    s_endpgm
2721;
2722; GFX8-LABEL: notsdot2_sext8:
2723; GFX8:       ; %bb.0: ; %entry
2724; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2725; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2726; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2727; GFX8-NEXT:    v_mov_b32_e32 v0, s4
2728; GFX8-NEXT:    v_mov_b32_e32 v1, s5
2729; GFX8-NEXT:    v_mov_b32_e32 v2, s6
2730; GFX8-NEXT:    v_mov_b32_e32 v3, s7
2731; GFX8-NEXT:    flat_load_ushort v2, v[2:3]
2732; GFX8-NEXT:    flat_load_ushort v0, v[0:1]
2733; GFX8-NEXT:    s_load_dword s2, s[0:1], 0x0
2734; GFX8-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(0)
2735; GFX8-NEXT:    v_bfe_i32 v3, v2, 0, 8
2736; GFX8-NEXT:    v_lshrrev_b16_e32 v2, 8, v2
2737; GFX8-NEXT:    s_waitcnt vmcnt(0)
2738; GFX8-NEXT:    v_bfe_i32 v1, v0, 0, 8
2739; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
2740; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 8
2741; GFX8-NEXT:    v_bfe_i32 v2, v2, 0, 8
2742; GFX8-NEXT:    v_mad_i32_i24 v0, v2, v0, s2
2743; GFX8-NEXT:    v_mad_i32_i24 v2, v3, v1, v0
2744; GFX8-NEXT:    v_mov_b32_e32 v0, s0
2745; GFX8-NEXT:    v_mov_b32_e32 v1, s1
2746; GFX8-NEXT:    flat_store_dword v[0:1], v2
2747; GFX8-NEXT:    s_endpgm
2748;
2749; GFX9-NODL-LABEL: notsdot2_sext8:
2750; GFX9-NODL:       ; %bb.0: ; %entry
2751; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2752; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2753; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2754; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s4
2755; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s5
2756; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s6
2757; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s7
2758; GFX9-NODL-NEXT:    global_load_ushort v2, v[2:3], off
2759; GFX9-NODL-NEXT:    global_load_ushort v0, v[0:1], off
2760; GFX9-NODL-NEXT:    s_load_dword s2, s[0:1], 0x0
2761; GFX9-NODL-NEXT:    s_waitcnt vmcnt(1)
2762; GFX9-NODL-NEXT:    v_bfe_i32 v3, v2, 0, 8
2763; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v2, 8, v2
2764; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
2765; GFX9-NODL-NEXT:    v_bfe_i32 v1, v0, 0, 8
2766; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
2767; GFX9-NODL-NEXT:    v_bfe_i32 v0, v0, 0, 8
2768; GFX9-NODL-NEXT:    v_bfe_i32 v2, v2, 0, 8
2769; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
2770; GFX9-NODL-NEXT:    v_mad_i32_i24 v0, v2, v0, s2
2771; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, v3, v1, v0
2772; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
2773; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
2774; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
2775; GFX9-NODL-NEXT:    s_endpgm
2776;
2777; GFX9-DL-LABEL: notsdot2_sext8:
2778; GFX9-DL:       ; %bb.0: ; %entry
2779; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2780; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2781; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2782; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s4
2783; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s5
2784; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s6
2785; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s7
2786; GFX9-DL-NEXT:    global_load_ushort v2, v[2:3], off
2787; GFX9-DL-NEXT:    global_load_ushort v0, v[0:1], off
2788; GFX9-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
2789; GFX9-DL-NEXT:    s_waitcnt vmcnt(1)
2790; GFX9-DL-NEXT:    v_bfe_i32 v3, v2, 0, 8
2791; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v2, 8, v2
2792; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
2793; GFX9-DL-NEXT:    v_bfe_i32 v1, v0, 0, 8
2794; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
2795; GFX9-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
2796; GFX9-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
2797; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
2798; GFX9-DL-NEXT:    v_mad_i32_i24 v0, v2, v0, s2
2799; GFX9-DL-NEXT:    v_mad_i32_i24 v2, v3, v1, v0
2800; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
2801; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
2802; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
2803; GFX9-DL-NEXT:    s_endpgm
2804;
2805; GFX10-DL-LABEL: notsdot2_sext8:
2806; GFX10-DL:       ; %bb.0: ; %entry
2807; GFX10-DL-NEXT:    s_clause 0x1
2808; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
2809; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
2810; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
2811; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2812; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s4
2813; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s5
2814; GFX10-DL-NEXT:    v_mov_b32_e32 v2, s6
2815; GFX10-DL-NEXT:    v_mov_b32_e32 v3, s7
2816; GFX10-DL-NEXT:    global_load_ushort v0, v[0:1], off
2817; GFX10-DL-NEXT:    global_load_ushort v1, v[2:3], off
2818; GFX10-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
2819; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
2820; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v2, 8, v0
2821; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
2822; GFX10-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, v1
2823; GFX10-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
2824; GFX10-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
2825; GFX10-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
2826; GFX10-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
2827; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
2828; GFX10-DL-NEXT:    v_mad_i32_i24 v2, v3, v2, s2
2829; GFX10-DL-NEXT:    v_mad_i32_i24 v2, v1, v0, v2
2830; GFX10-DL-NEXT:    v_mov_b32_e32 v0, s0
2831; GFX10-DL-NEXT:    v_mov_b32_e32 v1, s1
2832; GFX10-DL-NEXT:    global_store_dword v[0:1], v2, off
2833; GFX10-DL-NEXT:    s_endpgm
2834                                          <2 x i8> addrspace(1)* %src2,
2835                                          i32 addrspace(1)* nocapture %dst) {
2836entry:
2837  %vec1 = load <2 x i8>, <2 x i8> addrspace(1)* %src1
2838  %vec2 = load <2 x i8>, <2 x i8> addrspace(1)* %src2
2839
2840  %s1.elt1 = extractelement <2 x i8> %vec1, i64 0
2841  %conv = sext i8 %s1.elt1 to i32
2842  %s2.elt1 = extractelement <2 x i8> %vec2, i64 0
2843  %conv2 = sext i8 %s2.elt1 to i32
2844  %mul1 = mul nuw i32 %conv2, %conv
2845
2846  %s1.elt2 = extractelement <2 x i8> %vec1, i64 1
2847  %conv3 = sext i8 %s1.elt2 to i32
2848  %s2.elt2 = extractelement <2 x i8> %vec2, i64 1
2849  %conv4 = sext i8 %s2.elt2 to i32
2850  %mul2 = mul nuw i32 %conv4, %conv3
2851
2852  %s3 = load i32, i32 addrspace(1)* %dst, align 4
2853  %add = add i32 %mul2, %s3
2854  %add6 = add i32 %add, %mul1
2855  store i32 %add6, i32 addrspace(1)* %dst, align 4
2856  ret void
2857}
2858