1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+f16c -verify-machineinstrs | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+f16c -verify-machineinstrs | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -verify-machineinstrs | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl -verify-machineinstrs | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512VL
6
7;
8; Half to Float
9;
10
11define float @cvt_i16_to_f32(i16 %a0) nounwind {
12; ALL-LABEL: cvt_i16_to_f32:
13; ALL:       # BB#0:
14; ALL-NEXT:    movswl %di, %eax
15; ALL-NEXT:    vmovd %eax, %xmm0
16; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
17; ALL-NEXT:    retq
18  %1 = bitcast i16 %a0 to half
19  %2 = fpext half %1 to float
20  ret float %2
21}
22
23define <4 x float> @cvt_4i16_to_4f32(<4 x i16> %a0) nounwind {
24; AVX1-LABEL: cvt_4i16_to_4f32:
25; AVX1:       # BB#0:
26; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
27; AVX1-NEXT:    vmovq %xmm0, %rax
28; AVX1-NEXT:    movq %rax, %rcx
29; AVX1-NEXT:    movq %rax, %rdx
30; AVX1-NEXT:    movswl %ax, %esi
31; AVX1-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
32; AVX1-NEXT:    shrl $16, %eax
33; AVX1-NEXT:    shrq $32, %rcx
34; AVX1-NEXT:    shrq $48, %rdx
35; AVX1-NEXT:    movswl %dx, %edx
36; AVX1-NEXT:    vmovd %edx, %xmm0
37; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
38; AVX1-NEXT:    movswl %cx, %ecx
39; AVX1-NEXT:    vmovd %ecx, %xmm1
40; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
41; AVX1-NEXT:    cwtl
42; AVX1-NEXT:    vmovd %eax, %xmm2
43; AVX1-NEXT:    vcvtph2ps %xmm2, %xmm2
44; AVX1-NEXT:    vmovd %esi, %xmm3
45; AVX1-NEXT:    vcvtph2ps %xmm3, %xmm3
46; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
47; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
48; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
49; AVX1-NEXT:    retq
50;
51; AVX2-LABEL: cvt_4i16_to_4f32:
52; AVX2:       # BB#0:
53; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
54; AVX2-NEXT:    vmovq %xmm0, %rax
55; AVX2-NEXT:    movq %rax, %rcx
56; AVX2-NEXT:    movq %rax, %rdx
57; AVX2-NEXT:    movswl %ax, %esi
58; AVX2-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
59; AVX2-NEXT:    shrl $16, %eax
60; AVX2-NEXT:    shrq $32, %rcx
61; AVX2-NEXT:    shrq $48, %rdx
62; AVX2-NEXT:    movswl %dx, %edx
63; AVX2-NEXT:    vmovd %edx, %xmm0
64; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
65; AVX2-NEXT:    movswl %cx, %ecx
66; AVX2-NEXT:    vmovd %ecx, %xmm1
67; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
68; AVX2-NEXT:    cwtl
69; AVX2-NEXT:    vmovd %eax, %xmm2
70; AVX2-NEXT:    vcvtph2ps %xmm2, %xmm2
71; AVX2-NEXT:    vmovd %esi, %xmm3
72; AVX2-NEXT:    vcvtph2ps %xmm3, %xmm3
73; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
74; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
75; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
76; AVX2-NEXT:    retq
77;
78; AVX512F-LABEL: cvt_4i16_to_4f32:
79; AVX512F:       # BB#0:
80; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
81; AVX512F-NEXT:    vmovq %xmm0, %rax
82; AVX512F-NEXT:    movq %rax, %rcx
83; AVX512F-NEXT:    movq %rax, %rdx
84; AVX512F-NEXT:    movswl %ax, %esi
85; AVX512F-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
86; AVX512F-NEXT:    shrl $16, %eax
87; AVX512F-NEXT:    shrq $32, %rcx
88; AVX512F-NEXT:    shrq $48, %rdx
89; AVX512F-NEXT:    movswl %dx, %edx
90; AVX512F-NEXT:    vmovd %edx, %xmm0
91; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
92; AVX512F-NEXT:    movswl %cx, %ecx
93; AVX512F-NEXT:    vmovd %ecx, %xmm1
94; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
95; AVX512F-NEXT:    cwtl
96; AVX512F-NEXT:    vmovd %eax, %xmm2
97; AVX512F-NEXT:    vcvtph2ps %xmm2, %xmm2
98; AVX512F-NEXT:    vmovd %esi, %xmm3
99; AVX512F-NEXT:    vcvtph2ps %xmm3, %xmm3
100; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
101; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
102; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
103; AVX512F-NEXT:    retq
104;
105; AVX512VL-LABEL: cvt_4i16_to_4f32:
106; AVX512VL:       # BB#0:
107; AVX512VL-NEXT:    vpmovdw %xmm0, -{{[0-9]+}}(%rsp)
108; AVX512VL-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
109; AVX512VL-NEXT:    movq %rax, %rcx
110; AVX512VL-NEXT:    movq %rax, %rdx
111; AVX512VL-NEXT:    movswl %ax, %esi
112; AVX512VL-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
113; AVX512VL-NEXT:    shrl $16, %eax
114; AVX512VL-NEXT:    shrq $32, %rcx
115; AVX512VL-NEXT:    shrq $48, %rdx
116; AVX512VL-NEXT:    movswl %dx, %edx
117; AVX512VL-NEXT:    vmovd %edx, %xmm0
118; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
119; AVX512VL-NEXT:    movswl %cx, %ecx
120; AVX512VL-NEXT:    vmovd %ecx, %xmm1
121; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
122; AVX512VL-NEXT:    cwtl
123; AVX512VL-NEXT:    vmovd %eax, %xmm2
124; AVX512VL-NEXT:    vcvtph2ps %xmm2, %xmm2
125; AVX512VL-NEXT:    vmovd %esi, %xmm3
126; AVX512VL-NEXT:    vcvtph2ps %xmm3, %xmm3
127; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
128; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
129; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
130; AVX512VL-NEXT:    retq
131  %1 = bitcast <4 x i16> %a0 to <4 x half>
132  %2 = fpext <4 x half> %1 to <4 x float>
133  ret <4 x float> %2
134}
135
136define <4 x float> @cvt_8i16_to_4f32(<8 x i16> %a0) nounwind {
137; AVX1-LABEL: cvt_8i16_to_4f32:
138; AVX1:       # BB#0:
139; AVX1-NEXT:    vmovq %xmm0, %rax
140; AVX1-NEXT:    movq %rax, %rcx
141; AVX1-NEXT:    movq %rax, %rdx
142; AVX1-NEXT:    movswl %ax, %esi
143; AVX1-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
144; AVX1-NEXT:    shrl $16, %eax
145; AVX1-NEXT:    shrq $32, %rcx
146; AVX1-NEXT:    shrq $48, %rdx
147; AVX1-NEXT:    movswl %dx, %edx
148; AVX1-NEXT:    vmovd %edx, %xmm0
149; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
150; AVX1-NEXT:    movswl %cx, %ecx
151; AVX1-NEXT:    vmovd %ecx, %xmm1
152; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
153; AVX1-NEXT:    cwtl
154; AVX1-NEXT:    vmovd %eax, %xmm2
155; AVX1-NEXT:    vcvtph2ps %xmm2, %xmm2
156; AVX1-NEXT:    vmovd %esi, %xmm3
157; AVX1-NEXT:    vcvtph2ps %xmm3, %xmm3
158; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
159; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
160; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
161; AVX1-NEXT:    retq
162;
163; AVX2-LABEL: cvt_8i16_to_4f32:
164; AVX2:       # BB#0:
165; AVX2-NEXT:    vmovq %xmm0, %rax
166; AVX2-NEXT:    movq %rax, %rcx
167; AVX2-NEXT:    movq %rax, %rdx
168; AVX2-NEXT:    movswl %ax, %esi
169; AVX2-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
170; AVX2-NEXT:    shrl $16, %eax
171; AVX2-NEXT:    shrq $32, %rcx
172; AVX2-NEXT:    shrq $48, %rdx
173; AVX2-NEXT:    movswl %dx, %edx
174; AVX2-NEXT:    vmovd %edx, %xmm0
175; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
176; AVX2-NEXT:    movswl %cx, %ecx
177; AVX2-NEXT:    vmovd %ecx, %xmm1
178; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
179; AVX2-NEXT:    cwtl
180; AVX2-NEXT:    vmovd %eax, %xmm2
181; AVX2-NEXT:    vcvtph2ps %xmm2, %xmm2
182; AVX2-NEXT:    vmovd %esi, %xmm3
183; AVX2-NEXT:    vcvtph2ps %xmm3, %xmm3
184; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
185; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
186; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
187; AVX2-NEXT:    retq
188;
189; AVX512F-LABEL: cvt_8i16_to_4f32:
190; AVX512F:       # BB#0:
191; AVX512F-NEXT:    vmovq %xmm0, %rax
192; AVX512F-NEXT:    movq %rax, %rcx
193; AVX512F-NEXT:    movq %rax, %rdx
194; AVX512F-NEXT:    movswl %ax, %esi
195; AVX512F-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
196; AVX512F-NEXT:    shrl $16, %eax
197; AVX512F-NEXT:    shrq $32, %rcx
198; AVX512F-NEXT:    shrq $48, %rdx
199; AVX512F-NEXT:    movswl %dx, %edx
200; AVX512F-NEXT:    vmovd %edx, %xmm0
201; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
202; AVX512F-NEXT:    movswl %cx, %ecx
203; AVX512F-NEXT:    vmovd %ecx, %xmm1
204; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
205; AVX512F-NEXT:    cwtl
206; AVX512F-NEXT:    vmovd %eax, %xmm2
207; AVX512F-NEXT:    vcvtph2ps %xmm2, %xmm2
208; AVX512F-NEXT:    vmovd %esi, %xmm3
209; AVX512F-NEXT:    vcvtph2ps %xmm3, %xmm3
210; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
211; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
212; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
213; AVX512F-NEXT:    retq
214;
215; AVX512VL-LABEL: cvt_8i16_to_4f32:
216; AVX512VL:       # BB#0:
217; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
218; AVX512VL-NEXT:    vpmovdw %xmm0, -{{[0-9]+}}(%rsp)
219; AVX512VL-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
220; AVX512VL-NEXT:    movq %rax, %rcx
221; AVX512VL-NEXT:    movq %rax, %rdx
222; AVX512VL-NEXT:    movswl %ax, %esi
223; AVX512VL-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
224; AVX512VL-NEXT:    shrl $16, %eax
225; AVX512VL-NEXT:    shrq $32, %rcx
226; AVX512VL-NEXT:    shrq $48, %rdx
227; AVX512VL-NEXT:    movswl %dx, %edx
228; AVX512VL-NEXT:    vmovd %edx, %xmm0
229; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
230; AVX512VL-NEXT:    movswl %cx, %ecx
231; AVX512VL-NEXT:    vmovd %ecx, %xmm1
232; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
233; AVX512VL-NEXT:    cwtl
234; AVX512VL-NEXT:    vmovd %eax, %xmm2
235; AVX512VL-NEXT:    vcvtph2ps %xmm2, %xmm2
236; AVX512VL-NEXT:    vmovd %esi, %xmm3
237; AVX512VL-NEXT:    vcvtph2ps %xmm3, %xmm3
238; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
239; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
240; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
241; AVX512VL-NEXT:    retq
242  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
243  %2 = bitcast <4 x i16> %1 to <4 x half>
244  %3 = fpext <4 x half> %2 to <4 x float>
245  ret <4 x float> %3
246}
247
248define <8 x float> @cvt_8i16_to_8f32(<8 x i16> %a0) nounwind {
249; ALL-LABEL: cvt_8i16_to_8f32:
250; ALL:       # BB#0:
251; ALL-NEXT:    vpextrq $1, %xmm0, %rdx
252; ALL-NEXT:    movq %rdx, %r8
253; ALL-NEXT:    movq %rdx, %r10
254; ALL-NEXT:    movswl %dx, %r9d
255; ALL-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<kill>
256; ALL-NEXT:    shrl $16, %edx
257; ALL-NEXT:    shrq $32, %r8
258; ALL-NEXT:    shrq $48, %r10
259; ALL-NEXT:    vmovq %xmm0, %rdi
260; ALL-NEXT:    movq %rdi, %rax
261; ALL-NEXT:    movq %rdi, %rsi
262; ALL-NEXT:    movswl %di, %ecx
263; ALL-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<kill>
264; ALL-NEXT:    shrl $16, %edi
265; ALL-NEXT:    shrq $32, %rax
266; ALL-NEXT:    shrq $48, %rsi
267; ALL-NEXT:    movswl %si, %esi
268; ALL-NEXT:    vmovd %esi, %xmm0
269; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
270; ALL-NEXT:    cwtl
271; ALL-NEXT:    vmovd %eax, %xmm1
272; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
273; ALL-NEXT:    movswl %di, %eax
274; ALL-NEXT:    vmovd %eax, %xmm2
275; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
276; ALL-NEXT:    vmovd %ecx, %xmm3
277; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
278; ALL-NEXT:    movswl %r10w, %eax
279; ALL-NEXT:    vmovd %eax, %xmm4
280; ALL-NEXT:    vcvtph2ps %xmm4, %xmm4
281; ALL-NEXT:    movswl %r8w, %eax
282; ALL-NEXT:    vmovd %eax, %xmm5
283; ALL-NEXT:    vcvtph2ps %xmm5, %xmm5
284; ALL-NEXT:    movswl %dx, %eax
285; ALL-NEXT:    vmovd %eax, %xmm6
286; ALL-NEXT:    vcvtph2ps %xmm6, %xmm6
287; ALL-NEXT:    vmovd %r9d, %xmm7
288; ALL-NEXT:    vcvtph2ps %xmm7, %xmm7
289; ALL-NEXT:    vinsertps {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[2,3]
290; ALL-NEXT:    vinsertps {{.*#+}} xmm5 = xmm6[0,1],xmm5[0],xmm6[3]
291; ALL-NEXT:    vinsertps {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[0]
292; ALL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
293; ALL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
294; ALL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
295; ALL-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
296; ALL-NEXT:    retq
297  %1 = bitcast <8 x i16> %a0 to <8 x half>
298  %2 = fpext <8 x half> %1 to <8 x float>
299  ret <8 x float> %2
300}
301
302define <16 x float> @cvt_16i16_to_16f32(<16 x i16> %a0) nounwind {
303; AVX1-LABEL: cvt_16i16_to_16f32:
304; AVX1:       # BB#0:
305; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
306; AVX1-NEXT:    vmovq %xmm4, %rax
307; AVX1-NEXT:    movq %rax, %rcx
308; AVX1-NEXT:    shrq $48, %rcx
309; AVX1-NEXT:    movswl %cx, %ecx
310; AVX1-NEXT:    vmovd %ecx, %xmm8
311; AVX1-NEXT:    movq %rax, %rcx
312; AVX1-NEXT:    shrq $32, %rcx
313; AVX1-NEXT:    movswl %cx, %ecx
314; AVX1-NEXT:    vmovd %ecx, %xmm9
315; AVX1-NEXT:    movswl %ax, %ecx
316; AVX1-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
317; AVX1-NEXT:    shrl $16, %eax
318; AVX1-NEXT:    cwtl
319; AVX1-NEXT:    vmovd %eax, %xmm10
320; AVX1-NEXT:    vpextrq $1, %xmm4, %rax
321; AVX1-NEXT:    vmovd %ecx, %xmm11
322; AVX1-NEXT:    movq %rax, %rcx
323; AVX1-NEXT:    shrq $48, %rcx
324; AVX1-NEXT:    movswl %cx, %ecx
325; AVX1-NEXT:    vmovd %ecx, %xmm12
326; AVX1-NEXT:    movq %rax, %rcx
327; AVX1-NEXT:    shrq $32, %rcx
328; AVX1-NEXT:    movswl %cx, %ecx
329; AVX1-NEXT:    vmovd %ecx, %xmm13
330; AVX1-NEXT:    movswl %ax, %ecx
331; AVX1-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
332; AVX1-NEXT:    shrl $16, %eax
333; AVX1-NEXT:    cwtl
334; AVX1-NEXT:    vmovd %eax, %xmm14
335; AVX1-NEXT:    vmovq %xmm0, %rax
336; AVX1-NEXT:    vmovd %ecx, %xmm15
337; AVX1-NEXT:    movq %rax, %rcx
338; AVX1-NEXT:    shrq $48, %rcx
339; AVX1-NEXT:    movswl %cx, %ecx
340; AVX1-NEXT:    vmovd %ecx, %xmm2
341; AVX1-NEXT:    movq %rax, %rcx
342; AVX1-NEXT:    shrq $32, %rcx
343; AVX1-NEXT:    movswl %cx, %ecx
344; AVX1-NEXT:    vmovd %ecx, %xmm3
345; AVX1-NEXT:    movswl %ax, %ecx
346; AVX1-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
347; AVX1-NEXT:    shrl $16, %eax
348; AVX1-NEXT:    cwtl
349; AVX1-NEXT:    vmovd %eax, %xmm4
350; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
351; AVX1-NEXT:    vmovd %ecx, %xmm0
352; AVX1-NEXT:    movq %rax, %rcx
353; AVX1-NEXT:    shrq $48, %rcx
354; AVX1-NEXT:    movswl %cx, %ecx
355; AVX1-NEXT:    vmovd %ecx, %xmm5
356; AVX1-NEXT:    movq %rax, %rcx
357; AVX1-NEXT:    shrq $32, %rcx
358; AVX1-NEXT:    movswl %cx, %ecx
359; AVX1-NEXT:    vmovd %ecx, %xmm6
360; AVX1-NEXT:    movl %eax, %ecx
361; AVX1-NEXT:    shrl $16, %ecx
362; AVX1-NEXT:    movswl %cx, %ecx
363; AVX1-NEXT:    vmovd %ecx, %xmm7
364; AVX1-NEXT:    cwtl
365; AVX1-NEXT:    vmovd %eax, %xmm1
366; AVX1-NEXT:    vcvtph2ps %xmm8, %xmm8
367; AVX1-NEXT:    vcvtph2ps %xmm9, %xmm9
368; AVX1-NEXT:    vcvtph2ps %xmm10, %xmm10
369; AVX1-NEXT:    vcvtph2ps %xmm11, %xmm11
370; AVX1-NEXT:    vcvtph2ps %xmm12, %xmm12
371; AVX1-NEXT:    vcvtph2ps %xmm13, %xmm13
372; AVX1-NEXT:    vcvtph2ps %xmm14, %xmm14
373; AVX1-NEXT:    vcvtph2ps %xmm15, %xmm15
374; AVX1-NEXT:    vcvtph2ps %xmm2, %xmm2
375; AVX1-NEXT:    vcvtph2ps %xmm3, %xmm3
376; AVX1-NEXT:    vcvtph2ps %xmm4, %xmm4
377; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
378; AVX1-NEXT:    vcvtph2ps %xmm5, %xmm5
379; AVX1-NEXT:    vcvtph2ps %xmm6, %xmm6
380; AVX1-NEXT:    vcvtph2ps %xmm7, %xmm7
381; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
382; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[2,3]
383; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm6[0],xmm1[3]
384; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[0]
385; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[2,3]
386; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm3[0],xmm0[3]
387; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[0]
388; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
389; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm15[0],xmm14[0],xmm15[2,3]
390; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm13[0],xmm1[3]
391; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm12[0]
392; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm11[0],xmm10[0],xmm11[2,3]
393; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
394; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
395; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
396; AVX1-NEXT:    retq
397;
398; AVX2-LABEL: cvt_16i16_to_16f32:
399; AVX2:       # BB#0:
400; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
401; AVX2-NEXT:    vmovq %xmm4, %rax
402; AVX2-NEXT:    movq %rax, %rcx
403; AVX2-NEXT:    shrq $48, %rcx
404; AVX2-NEXT:    movswl %cx, %ecx
405; AVX2-NEXT:    vmovd %ecx, %xmm8
406; AVX2-NEXT:    movq %rax, %rcx
407; AVX2-NEXT:    shrq $32, %rcx
408; AVX2-NEXT:    movswl %cx, %ecx
409; AVX2-NEXT:    vmovd %ecx, %xmm9
410; AVX2-NEXT:    movswl %ax, %ecx
411; AVX2-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
412; AVX2-NEXT:    shrl $16, %eax
413; AVX2-NEXT:    cwtl
414; AVX2-NEXT:    vmovd %eax, %xmm10
415; AVX2-NEXT:    vpextrq $1, %xmm4, %rax
416; AVX2-NEXT:    vmovd %ecx, %xmm11
417; AVX2-NEXT:    movq %rax, %rcx
418; AVX2-NEXT:    shrq $48, %rcx
419; AVX2-NEXT:    movswl %cx, %ecx
420; AVX2-NEXT:    vmovd %ecx, %xmm12
421; AVX2-NEXT:    movq %rax, %rcx
422; AVX2-NEXT:    shrq $32, %rcx
423; AVX2-NEXT:    movswl %cx, %ecx
424; AVX2-NEXT:    vmovd %ecx, %xmm13
425; AVX2-NEXT:    movswl %ax, %ecx
426; AVX2-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
427; AVX2-NEXT:    shrl $16, %eax
428; AVX2-NEXT:    cwtl
429; AVX2-NEXT:    vmovd %eax, %xmm14
430; AVX2-NEXT:    vmovq %xmm0, %rax
431; AVX2-NEXT:    vmovd %ecx, %xmm15
432; AVX2-NEXT:    movq %rax, %rcx
433; AVX2-NEXT:    shrq $48, %rcx
434; AVX2-NEXT:    movswl %cx, %ecx
435; AVX2-NEXT:    vmovd %ecx, %xmm2
436; AVX2-NEXT:    movq %rax, %rcx
437; AVX2-NEXT:    shrq $32, %rcx
438; AVX2-NEXT:    movswl %cx, %ecx
439; AVX2-NEXT:    vmovd %ecx, %xmm3
440; AVX2-NEXT:    movswl %ax, %ecx
441; AVX2-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
442; AVX2-NEXT:    shrl $16, %eax
443; AVX2-NEXT:    cwtl
444; AVX2-NEXT:    vmovd %eax, %xmm4
445; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
446; AVX2-NEXT:    vmovd %ecx, %xmm0
447; AVX2-NEXT:    movq %rax, %rcx
448; AVX2-NEXT:    shrq $48, %rcx
449; AVX2-NEXT:    movswl %cx, %ecx
450; AVX2-NEXT:    vmovd %ecx, %xmm5
451; AVX2-NEXT:    movq %rax, %rcx
452; AVX2-NEXT:    shrq $32, %rcx
453; AVX2-NEXT:    movswl %cx, %ecx
454; AVX2-NEXT:    vmovd %ecx, %xmm6
455; AVX2-NEXT:    movl %eax, %ecx
456; AVX2-NEXT:    shrl $16, %ecx
457; AVX2-NEXT:    movswl %cx, %ecx
458; AVX2-NEXT:    vmovd %ecx, %xmm7
459; AVX2-NEXT:    cwtl
460; AVX2-NEXT:    vmovd %eax, %xmm1
461; AVX2-NEXT:    vcvtph2ps %xmm8, %xmm8
462; AVX2-NEXT:    vcvtph2ps %xmm9, %xmm9
463; AVX2-NEXT:    vcvtph2ps %xmm10, %xmm10
464; AVX2-NEXT:    vcvtph2ps %xmm11, %xmm11
465; AVX2-NEXT:    vcvtph2ps %xmm12, %xmm12
466; AVX2-NEXT:    vcvtph2ps %xmm13, %xmm13
467; AVX2-NEXT:    vcvtph2ps %xmm14, %xmm14
468; AVX2-NEXT:    vcvtph2ps %xmm15, %xmm15
469; AVX2-NEXT:    vcvtph2ps %xmm2, %xmm2
470; AVX2-NEXT:    vcvtph2ps %xmm3, %xmm3
471; AVX2-NEXT:    vcvtph2ps %xmm4, %xmm4
472; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
473; AVX2-NEXT:    vcvtph2ps %xmm5, %xmm5
474; AVX2-NEXT:    vcvtph2ps %xmm6, %xmm6
475; AVX2-NEXT:    vcvtph2ps %xmm7, %xmm7
476; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
477; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[2,3]
478; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm6[0],xmm1[3]
479; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[0]
480; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[2,3]
481; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm3[0],xmm0[3]
482; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[0]
483; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
484; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm15[0],xmm14[0],xmm15[2,3]
485; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm13[0],xmm1[3]
486; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm12[0]
487; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm11[0],xmm10[0],xmm11[2,3]
488; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
489; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
490; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
491; AVX2-NEXT:    retq
492;
493; AVX512F-LABEL: cvt_16i16_to_16f32:
494; AVX512F:       # BB#0:
495; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm10
496; AVX512F-NEXT:    vmovq %xmm0, %rax
497; AVX512F-NEXT:    movq %rax, %rcx
498; AVX512F-NEXT:    shrq $48, %rcx
499; AVX512F-NEXT:    movswl %cx, %ecx
500; AVX512F-NEXT:    vmovd %ecx, %xmm8
501; AVX512F-NEXT:    movq %rax, %rcx
502; AVX512F-NEXT:    shrq $32, %rcx
503; AVX512F-NEXT:    movswl %cx, %ecx
504; AVX512F-NEXT:    vmovd %ecx, %xmm9
505; AVX512F-NEXT:    movswl %ax, %ecx
506; AVX512F-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
507; AVX512F-NEXT:    shrl $16, %eax
508; AVX512F-NEXT:    cwtl
509; AVX512F-NEXT:    vmovd %eax, %xmm11
510; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
511; AVX512F-NEXT:    vmovd %ecx, %xmm12
512; AVX512F-NEXT:    movq %rax, %rcx
513; AVX512F-NEXT:    shrq $48, %rcx
514; AVX512F-NEXT:    movswl %cx, %ecx
515; AVX512F-NEXT:    vmovd %ecx, %xmm13
516; AVX512F-NEXT:    movq %rax, %rcx
517; AVX512F-NEXT:    shrq $32, %rcx
518; AVX512F-NEXT:    movswl %cx, %ecx
519; AVX512F-NEXT:    vmovd %ecx, %xmm14
520; AVX512F-NEXT:    movswl %ax, %ecx
521; AVX512F-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
522; AVX512F-NEXT:    shrl $16, %eax
523; AVX512F-NEXT:    cwtl
524; AVX512F-NEXT:    vmovd %eax, %xmm15
525; AVX512F-NEXT:    vmovq %xmm10, %rax
526; AVX512F-NEXT:    vmovd %ecx, %xmm2
527; AVX512F-NEXT:    movq %rax, %rcx
528; AVX512F-NEXT:    shrq $48, %rcx
529; AVX512F-NEXT:    movswl %cx, %ecx
530; AVX512F-NEXT:    vmovd %ecx, %xmm3
531; AVX512F-NEXT:    movq %rax, %rcx
532; AVX512F-NEXT:    shrq $32, %rcx
533; AVX512F-NEXT:    movswl %cx, %ecx
534; AVX512F-NEXT:    vmovd %ecx, %xmm1
535; AVX512F-NEXT:    movswl %ax, %ecx
536; AVX512F-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
537; AVX512F-NEXT:    shrl $16, %eax
538; AVX512F-NEXT:    cwtl
539; AVX512F-NEXT:    vmovd %eax, %xmm4
540; AVX512F-NEXT:    vpextrq $1, %xmm10, %rax
541; AVX512F-NEXT:    vmovd %ecx, %xmm10
542; AVX512F-NEXT:    movq %rax, %rcx
543; AVX512F-NEXT:    shrq $48, %rcx
544; AVX512F-NEXT:    movswl %cx, %ecx
545; AVX512F-NEXT:    vmovd %ecx, %xmm5
546; AVX512F-NEXT:    movq %rax, %rcx
547; AVX512F-NEXT:    shrq $32, %rcx
548; AVX512F-NEXT:    movswl %cx, %ecx
549; AVX512F-NEXT:    vmovd %ecx, %xmm6
550; AVX512F-NEXT:    movl %eax, %ecx
551; AVX512F-NEXT:    shrl $16, %ecx
552; AVX512F-NEXT:    movswl %cx, %ecx
553; AVX512F-NEXT:    vmovd %ecx, %xmm7
554; AVX512F-NEXT:    cwtl
555; AVX512F-NEXT:    vmovd %eax, %xmm0
556; AVX512F-NEXT:    vcvtph2ps %xmm8, %xmm8
557; AVX512F-NEXT:    vcvtph2ps %xmm9, %xmm9
558; AVX512F-NEXT:    vcvtph2ps %xmm11, %xmm11
559; AVX512F-NEXT:    vcvtph2ps %xmm12, %xmm12
560; AVX512F-NEXT:    vcvtph2ps %xmm13, %xmm13
561; AVX512F-NEXT:    vcvtph2ps %xmm14, %xmm14
562; AVX512F-NEXT:    vcvtph2ps %xmm15, %xmm15
563; AVX512F-NEXT:    vcvtph2ps %xmm2, %xmm2
564; AVX512F-NEXT:    vcvtph2ps %xmm3, %xmm3
565; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
566; AVX512F-NEXT:    vcvtph2ps %xmm4, %xmm4
567; AVX512F-NEXT:    vcvtph2ps %xmm10, %xmm10
568; AVX512F-NEXT:    vcvtph2ps %xmm5, %xmm5
569; AVX512F-NEXT:    vcvtph2ps %xmm6, %xmm6
570; AVX512F-NEXT:    vcvtph2ps %xmm7, %xmm7
571; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
572; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[2,3]
573; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm6[0],xmm0[3]
574; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm5[0]
575; AVX512F-NEXT:    vinsertps {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[2,3]
576; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm4[0,1],xmm1[0],xmm4[3]
577; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[0]
578; AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
579; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm15[0],xmm2[2,3]
580; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm14[0],xmm1[3]
581; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm13[0]
582; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm12[0],xmm11[0],xmm12[2,3]
583; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
584; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
585; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
586; AVX512F-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
587; AVX512F-NEXT:    retq
588;
589; AVX512VL-LABEL: cvt_16i16_to_16f32:
590; AVX512VL:       # BB#0:
591; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm10
592; AVX512VL-NEXT:    vmovq %xmm0, %rax
593; AVX512VL-NEXT:    movq %rax, %rcx
594; AVX512VL-NEXT:    shrq $48, %rcx
595; AVX512VL-NEXT:    movswl %cx, %ecx
596; AVX512VL-NEXT:    vmovd %ecx, %xmm8
597; AVX512VL-NEXT:    movq %rax, %rcx
598; AVX512VL-NEXT:    shrq $32, %rcx
599; AVX512VL-NEXT:    movswl %cx, %ecx
600; AVX512VL-NEXT:    vmovd %ecx, %xmm9
601; AVX512VL-NEXT:    movswl %ax, %ecx
602; AVX512VL-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
603; AVX512VL-NEXT:    shrl $16, %eax
604; AVX512VL-NEXT:    cwtl
605; AVX512VL-NEXT:    vmovd %eax, %xmm11
606; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
607; AVX512VL-NEXT:    vmovd %ecx, %xmm12
608; AVX512VL-NEXT:    movq %rax, %rcx
609; AVX512VL-NEXT:    shrq $48, %rcx
610; AVX512VL-NEXT:    movswl %cx, %ecx
611; AVX512VL-NEXT:    vmovd %ecx, %xmm13
612; AVX512VL-NEXT:    movq %rax, %rcx
613; AVX512VL-NEXT:    shrq $32, %rcx
614; AVX512VL-NEXT:    movswl %cx, %ecx
615; AVX512VL-NEXT:    vmovd %ecx, %xmm14
616; AVX512VL-NEXT:    movswl %ax, %ecx
617; AVX512VL-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
618; AVX512VL-NEXT:    shrl $16, %eax
619; AVX512VL-NEXT:    cwtl
620; AVX512VL-NEXT:    vmovd %eax, %xmm15
621; AVX512VL-NEXT:    vmovq %xmm10, %rax
622; AVX512VL-NEXT:    vmovd %ecx, %xmm16
623; AVX512VL-NEXT:    movq %rax, %rcx
624; AVX512VL-NEXT:    shrq $48, %rcx
625; AVX512VL-NEXT:    movswl %cx, %ecx
626; AVX512VL-NEXT:    vmovd %ecx, %xmm17
627; AVX512VL-NEXT:    movq %rax, %rcx
628; AVX512VL-NEXT:    shrq $32, %rcx
629; AVX512VL-NEXT:    movswl %cx, %ecx
630; AVX512VL-NEXT:    vmovd %ecx, %xmm18
631; AVX512VL-NEXT:    movswl %ax, %ecx
632; AVX512VL-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
633; AVX512VL-NEXT:    shrl $16, %eax
634; AVX512VL-NEXT:    cwtl
635; AVX512VL-NEXT:    vmovd %eax, %xmm19
636; AVX512VL-NEXT:    vpextrq $1, %xmm10, %rax
637; AVX512VL-NEXT:    vmovd %ecx, %xmm10
638; AVX512VL-NEXT:    movq %rax, %rcx
639; AVX512VL-NEXT:    shrq $48, %rcx
640; AVX512VL-NEXT:    movswl %cx, %ecx
641; AVX512VL-NEXT:    vmovd %ecx, %xmm20
642; AVX512VL-NEXT:    movq %rax, %rcx
643; AVX512VL-NEXT:    shrq $32, %rcx
644; AVX512VL-NEXT:    movswl %cx, %ecx
645; AVX512VL-NEXT:    vmovd %ecx, %xmm21
646; AVX512VL-NEXT:    movl %eax, %ecx
647; AVX512VL-NEXT:    shrl $16, %ecx
648; AVX512VL-NEXT:    movswl %cx, %ecx
649; AVX512VL-NEXT:    vmovd %ecx, %xmm22
650; AVX512VL-NEXT:    cwtl
651; AVX512VL-NEXT:    vmovd %eax, %xmm2
652; AVX512VL-NEXT:    vcvtph2ps %xmm8, %xmm8
653; AVX512VL-NEXT:    vcvtph2ps %xmm9, %xmm9
654; AVX512VL-NEXT:    vcvtph2ps %xmm11, %xmm11
655; AVX512VL-NEXT:    vcvtph2ps %xmm12, %xmm12
656; AVX512VL-NEXT:    vcvtph2ps %xmm13, %xmm13
657; AVX512VL-NEXT:    vcvtph2ps %xmm14, %xmm14
658; AVX512VL-NEXT:    vcvtph2ps %xmm15, %xmm15
659; AVX512VL-NEXT:    vcvtph2ps %xmm16, %xmm16
660; AVX512VL-NEXT:    vcvtph2ps %xmm17, %xmm4
661; AVX512VL-NEXT:    vcvtph2ps %xmm18, %xmm0
662; AVX512VL-NEXT:    vcvtph2ps %xmm19, %xmm5
663; AVX512VL-NEXT:    vcvtph2ps %xmm10, %xmm7
664; AVX512VL-NEXT:    vcvtph2ps %xmm20, %xmm3
665; AVX512VL-NEXT:    vcvtph2ps %xmm21, %xmm6
666; AVX512VL-NEXT:    vcvtph2ps %xmm22, %xmm1
667; AVX512VL-NEXT:    vcvtph2ps %xmm2, %xmm2
668; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
669; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm6[0],xmm1[3]
670; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[0]
671; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm7[0],xmm5[0],xmm7[2,3]
672; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1],xmm0[0],xmm2[3]
673; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm4[0]
674; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
675; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm16[0],xmm15[0],xmm16[2,3]
676; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm14[0],xmm1[3]
677; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm13[0]
678; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm12[0],xmm11[0],xmm12[2,3]
679; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
680; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
681; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
682; AVX512VL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
683; AVX512VL-NEXT:    retq
684  %1 = bitcast <16 x i16> %a0 to <16 x half>
685  %2 = fpext <16 x half> %1 to <16 x float>
686  ret <16 x float> %2
687}
688
689;
690; Half to Float (Load)
691;
692
693define float @load_cvt_i16_to_f32(i16* %a0) nounwind {
694; ALL-LABEL: load_cvt_i16_to_f32:
695; ALL:       # BB#0:
696; ALL-NEXT:    movswl (%rdi), %eax
697; ALL-NEXT:    vmovd %eax, %xmm0
698; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
699; ALL-NEXT:    retq
700  %1 = load i16, i16* %a0
701  %2 = bitcast i16 %1 to half
702  %3 = fpext half %2 to float
703  ret float %3
704}
705
706define <4 x float> @load_cvt_4i16_to_4f32(<4 x i16>* %a0) nounwind {
707; ALL-LABEL: load_cvt_4i16_to_4f32:
708; ALL:       # BB#0:
709; ALL-NEXT:    movswl 6(%rdi), %eax
710; ALL-NEXT:    vmovd %eax, %xmm0
711; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
712; ALL-NEXT:    movswl 4(%rdi), %eax
713; ALL-NEXT:    vmovd %eax, %xmm1
714; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
715; ALL-NEXT:    movswl (%rdi), %eax
716; ALL-NEXT:    vmovd %eax, %xmm2
717; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
718; ALL-NEXT:    movswl 2(%rdi), %eax
719; ALL-NEXT:    vmovd %eax, %xmm3
720; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
721; ALL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
722; ALL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
723; ALL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
724; ALL-NEXT:    retq
725  %1 = load <4 x i16>, <4 x i16>* %a0
726  %2 = bitcast <4 x i16> %1 to <4 x half>
727  %3 = fpext <4 x half> %2 to <4 x float>
728  ret <4 x float> %3
729}
730
731define <4 x float> @load_cvt_8i16_to_4f32(<8 x i16>* %a0) nounwind {
732; AVX1-LABEL: load_cvt_8i16_to_4f32:
733; AVX1:       # BB#0:
734; AVX1-NEXT:    movq (%rdi), %rax
735; AVX1-NEXT:    movq %rax, %rcx
736; AVX1-NEXT:    movq %rax, %rdx
737; AVX1-NEXT:    movswl %ax, %esi
738; AVX1-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
739; AVX1-NEXT:    shrl $16, %eax
740; AVX1-NEXT:    shrq $32, %rcx
741; AVX1-NEXT:    shrq $48, %rdx
742; AVX1-NEXT:    movswl %dx, %edx
743; AVX1-NEXT:    vmovd %edx, %xmm0
744; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
745; AVX1-NEXT:    movswl %cx, %ecx
746; AVX1-NEXT:    vmovd %ecx, %xmm1
747; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
748; AVX1-NEXT:    cwtl
749; AVX1-NEXT:    vmovd %eax, %xmm2
750; AVX1-NEXT:    vcvtph2ps %xmm2, %xmm2
751; AVX1-NEXT:    vmovd %esi, %xmm3
752; AVX1-NEXT:    vcvtph2ps %xmm3, %xmm3
753; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
754; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
755; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
756; AVX1-NEXT:    retq
757;
758; AVX2-LABEL: load_cvt_8i16_to_4f32:
759; AVX2:       # BB#0:
760; AVX2-NEXT:    movq (%rdi), %rax
761; AVX2-NEXT:    movq %rax, %rcx
762; AVX2-NEXT:    movq %rax, %rdx
763; AVX2-NEXT:    movswl %ax, %esi
764; AVX2-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
765; AVX2-NEXT:    shrl $16, %eax
766; AVX2-NEXT:    shrq $32, %rcx
767; AVX2-NEXT:    shrq $48, %rdx
768; AVX2-NEXT:    movswl %dx, %edx
769; AVX2-NEXT:    vmovd %edx, %xmm0
770; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
771; AVX2-NEXT:    movswl %cx, %ecx
772; AVX2-NEXT:    vmovd %ecx, %xmm1
773; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
774; AVX2-NEXT:    cwtl
775; AVX2-NEXT:    vmovd %eax, %xmm2
776; AVX2-NEXT:    vcvtph2ps %xmm2, %xmm2
777; AVX2-NEXT:    vmovd %esi, %xmm3
778; AVX2-NEXT:    vcvtph2ps %xmm3, %xmm3
779; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
780; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
781; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
782; AVX2-NEXT:    retq
783;
784; AVX512F-LABEL: load_cvt_8i16_to_4f32:
785; AVX512F:       # BB#0:
786; AVX512F-NEXT:    movq (%rdi), %rax
787; AVX512F-NEXT:    movq %rax, %rcx
788; AVX512F-NEXT:    movq %rax, %rdx
789; AVX512F-NEXT:    movswl %ax, %esi
790; AVX512F-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
791; AVX512F-NEXT:    shrl $16, %eax
792; AVX512F-NEXT:    shrq $32, %rcx
793; AVX512F-NEXT:    shrq $48, %rdx
794; AVX512F-NEXT:    movswl %dx, %edx
795; AVX512F-NEXT:    vmovd %edx, %xmm0
796; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
797; AVX512F-NEXT:    movswl %cx, %ecx
798; AVX512F-NEXT:    vmovd %ecx, %xmm1
799; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
800; AVX512F-NEXT:    cwtl
801; AVX512F-NEXT:    vmovd %eax, %xmm2
802; AVX512F-NEXT:    vcvtph2ps %xmm2, %xmm2
803; AVX512F-NEXT:    vmovd %esi, %xmm3
804; AVX512F-NEXT:    vcvtph2ps %xmm3, %xmm3
805; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
806; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
807; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
808; AVX512F-NEXT:    retq
809;
810; AVX512VL-LABEL: load_cvt_8i16_to_4f32:
811; AVX512VL:       # BB#0:
812; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
813; AVX512VL-NEXT:    vpmovdw %xmm0, -{{[0-9]+}}(%rsp)
814; AVX512VL-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
815; AVX512VL-NEXT:    movq %rax, %rcx
816; AVX512VL-NEXT:    movq %rax, %rdx
817; AVX512VL-NEXT:    movswl %ax, %esi
818; AVX512VL-NEXT:    # kill: %EAX<def> %EAX<kill> %RAX<kill>
819; AVX512VL-NEXT:    shrl $16, %eax
820; AVX512VL-NEXT:    shrq $32, %rcx
821; AVX512VL-NEXT:    shrq $48, %rdx
822; AVX512VL-NEXT:    movswl %dx, %edx
823; AVX512VL-NEXT:    vmovd %edx, %xmm0
824; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
825; AVX512VL-NEXT:    movswl %cx, %ecx
826; AVX512VL-NEXT:    vmovd %ecx, %xmm1
827; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
828; AVX512VL-NEXT:    cwtl
829; AVX512VL-NEXT:    vmovd %eax, %xmm2
830; AVX512VL-NEXT:    vcvtph2ps %xmm2, %xmm2
831; AVX512VL-NEXT:    vmovd %esi, %xmm3
832; AVX512VL-NEXT:    vcvtph2ps %xmm3, %xmm3
833; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
834; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
835; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
836; AVX512VL-NEXT:    retq
837  %1 = load <8 x i16>, <8 x i16>* %a0
838  %2 = shufflevector <8 x i16> %1, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
839  %3 = bitcast <4 x i16> %2 to <4 x half>
840  %4 = fpext <4 x half> %3 to <4 x float>
841  ret <4 x float> %4
842}
843
844define <8 x float> @load_cvt_8i16_to_8f32(<8 x i16>* %a0) nounwind {
845; ALL-LABEL: load_cvt_8i16_to_8f32:
846; ALL:       # BB#0:
847; ALL-NEXT:    movswl 6(%rdi), %eax
848; ALL-NEXT:    vmovd %eax, %xmm0
849; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
850; ALL-NEXT:    movswl 4(%rdi), %eax
851; ALL-NEXT:    vmovd %eax, %xmm1
852; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
853; ALL-NEXT:    movswl (%rdi), %eax
854; ALL-NEXT:    vmovd %eax, %xmm2
855; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
856; ALL-NEXT:    movswl 2(%rdi), %eax
857; ALL-NEXT:    vmovd %eax, %xmm3
858; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
859; ALL-NEXT:    movswl 14(%rdi), %eax
860; ALL-NEXT:    vmovd %eax, %xmm4
861; ALL-NEXT:    vcvtph2ps %xmm4, %xmm4
862; ALL-NEXT:    movswl 12(%rdi), %eax
863; ALL-NEXT:    vmovd %eax, %xmm5
864; ALL-NEXT:    vcvtph2ps %xmm5, %xmm5
865; ALL-NEXT:    movswl 8(%rdi), %eax
866; ALL-NEXT:    vmovd %eax, %xmm6
867; ALL-NEXT:    vcvtph2ps %xmm6, %xmm6
868; ALL-NEXT:    movswl 10(%rdi), %eax
869; ALL-NEXT:    vmovd %eax, %xmm7
870; ALL-NEXT:    vcvtph2ps %xmm7, %xmm7
871; ALL-NEXT:    vinsertps {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[2,3]
872; ALL-NEXT:    vinsertps {{.*#+}} xmm5 = xmm6[0,1],xmm5[0],xmm6[3]
873; ALL-NEXT:    vinsertps {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[0]
874; ALL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
875; ALL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
876; ALL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
877; ALL-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
878; ALL-NEXT:    retq
879  %1 = load <8 x i16>, <8 x i16>* %a0
880  %2 = bitcast <8 x i16> %1 to <8 x half>
881  %3 = fpext <8 x half> %2 to <8 x float>
882  ret <8 x float> %3
883}
884
885define <16 x float> @load_cvt_16i16_to_16f32(<16 x i16>* %a0) nounwind {
886; AVX1-LABEL: load_cvt_16i16_to_16f32:
887; AVX1:       # BB#0:
888; AVX1-NEXT:    movswl 22(%rdi), %eax
889; AVX1-NEXT:    vmovd %eax, %xmm0
890; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm8
891; AVX1-NEXT:    movswl 20(%rdi), %eax
892; AVX1-NEXT:    vmovd %eax, %xmm0
893; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm9
894; AVX1-NEXT:    movswl 16(%rdi), %eax
895; AVX1-NEXT:    vmovd %eax, %xmm0
896; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm10
897; AVX1-NEXT:    movswl 18(%rdi), %eax
898; AVX1-NEXT:    vmovd %eax, %xmm0
899; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm11
900; AVX1-NEXT:    movswl 30(%rdi), %eax
901; AVX1-NEXT:    vmovd %eax, %xmm0
902; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm12
903; AVX1-NEXT:    movswl 28(%rdi), %eax
904; AVX1-NEXT:    vmovd %eax, %xmm0
905; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm13
906; AVX1-NEXT:    movswl 24(%rdi), %eax
907; AVX1-NEXT:    vmovd %eax, %xmm0
908; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm14
909; AVX1-NEXT:    movswl 26(%rdi), %eax
910; AVX1-NEXT:    vmovd %eax, %xmm0
911; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm15
912; AVX1-NEXT:    movswl 6(%rdi), %eax
913; AVX1-NEXT:    vmovd %eax, %xmm0
914; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
915; AVX1-NEXT:    movswl 4(%rdi), %eax
916; AVX1-NEXT:    vmovd %eax, %xmm2
917; AVX1-NEXT:    vcvtph2ps %xmm2, %xmm2
918; AVX1-NEXT:    movswl (%rdi), %eax
919; AVX1-NEXT:    vmovd %eax, %xmm3
920; AVX1-NEXT:    vcvtph2ps %xmm3, %xmm3
921; AVX1-NEXT:    movswl 2(%rdi), %eax
922; AVX1-NEXT:    vmovd %eax, %xmm4
923; AVX1-NEXT:    vcvtph2ps %xmm4, %xmm4
924; AVX1-NEXT:    movswl 14(%rdi), %eax
925; AVX1-NEXT:    vmovd %eax, %xmm5
926; AVX1-NEXT:    vcvtph2ps %xmm5, %xmm5
927; AVX1-NEXT:    movswl 12(%rdi), %eax
928; AVX1-NEXT:    vmovd %eax, %xmm6
929; AVX1-NEXT:    vcvtph2ps %xmm6, %xmm6
930; AVX1-NEXT:    movswl 8(%rdi), %eax
931; AVX1-NEXT:    vmovd %eax, %xmm7
932; AVX1-NEXT:    vcvtph2ps %xmm7, %xmm7
933; AVX1-NEXT:    movswl 10(%rdi), %eax
934; AVX1-NEXT:    vmovd %eax, %xmm1
935; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
936; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm7[0],xmm1[0],xmm7[2,3]
937; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm6[0],xmm1[3]
938; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[0]
939; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[2,3]
940; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0,1],xmm2[0],xmm3[3]
941; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
942; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
943; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm14[0],xmm15[0],xmm14[2,3]
944; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm13[0],xmm1[3]
945; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm12[0]
946; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm10[0],xmm11[0],xmm10[2,3]
947; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
948; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
949; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
950; AVX1-NEXT:    retq
951;
952; AVX2-LABEL: load_cvt_16i16_to_16f32:
953; AVX2:       # BB#0:
954; AVX2-NEXT:    movswl 22(%rdi), %eax
955; AVX2-NEXT:    vmovd %eax, %xmm0
956; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm8
957; AVX2-NEXT:    movswl 20(%rdi), %eax
958; AVX2-NEXT:    vmovd %eax, %xmm0
959; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm9
960; AVX2-NEXT:    movswl 16(%rdi), %eax
961; AVX2-NEXT:    vmovd %eax, %xmm0
962; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm10
963; AVX2-NEXT:    movswl 18(%rdi), %eax
964; AVX2-NEXT:    vmovd %eax, %xmm0
965; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm11
966; AVX2-NEXT:    movswl 30(%rdi), %eax
967; AVX2-NEXT:    vmovd %eax, %xmm0
968; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm12
969; AVX2-NEXT:    movswl 28(%rdi), %eax
970; AVX2-NEXT:    vmovd %eax, %xmm0
971; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm13
972; AVX2-NEXT:    movswl 24(%rdi), %eax
973; AVX2-NEXT:    vmovd %eax, %xmm0
974; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm14
975; AVX2-NEXT:    movswl 26(%rdi), %eax
976; AVX2-NEXT:    vmovd %eax, %xmm0
977; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm15
978; AVX2-NEXT:    movswl 6(%rdi), %eax
979; AVX2-NEXT:    vmovd %eax, %xmm0
980; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
981; AVX2-NEXT:    movswl 4(%rdi), %eax
982; AVX2-NEXT:    vmovd %eax, %xmm2
983; AVX2-NEXT:    vcvtph2ps %xmm2, %xmm2
984; AVX2-NEXT:    movswl (%rdi), %eax
985; AVX2-NEXT:    vmovd %eax, %xmm3
986; AVX2-NEXT:    vcvtph2ps %xmm3, %xmm3
987; AVX2-NEXT:    movswl 2(%rdi), %eax
988; AVX2-NEXT:    vmovd %eax, %xmm4
989; AVX2-NEXT:    vcvtph2ps %xmm4, %xmm4
990; AVX2-NEXT:    movswl 14(%rdi), %eax
991; AVX2-NEXT:    vmovd %eax, %xmm5
992; AVX2-NEXT:    vcvtph2ps %xmm5, %xmm5
993; AVX2-NEXT:    movswl 12(%rdi), %eax
994; AVX2-NEXT:    vmovd %eax, %xmm6
995; AVX2-NEXT:    vcvtph2ps %xmm6, %xmm6
996; AVX2-NEXT:    movswl 8(%rdi), %eax
997; AVX2-NEXT:    vmovd %eax, %xmm7
998; AVX2-NEXT:    vcvtph2ps %xmm7, %xmm7
999; AVX2-NEXT:    movswl 10(%rdi), %eax
1000; AVX2-NEXT:    vmovd %eax, %xmm1
1001; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
1002; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm7[0],xmm1[0],xmm7[2,3]
1003; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm6[0],xmm1[3]
1004; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[0]
1005; AVX2-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[2,3]
1006; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0,1],xmm2[0],xmm3[3]
1007; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
1008; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1009; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm14[0],xmm15[0],xmm14[2,3]
1010; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm13[0],xmm1[3]
1011; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm12[0]
1012; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm10[0],xmm11[0],xmm10[2,3]
1013; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
1014; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
1015; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
1016; AVX2-NEXT:    retq
1017;
1018; AVX512F-LABEL: load_cvt_16i16_to_16f32:
1019; AVX512F:       # BB#0:
1020; AVX512F-NEXT:    movswl 6(%rdi), %eax
1021; AVX512F-NEXT:    vmovd %eax, %xmm0
1022; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm8
1023; AVX512F-NEXT:    movswl 4(%rdi), %eax
1024; AVX512F-NEXT:    vmovd %eax, %xmm0
1025; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm9
1026; AVX512F-NEXT:    movswl (%rdi), %eax
1027; AVX512F-NEXT:    vmovd %eax, %xmm0
1028; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm10
1029; AVX512F-NEXT:    movswl 2(%rdi), %eax
1030; AVX512F-NEXT:    vmovd %eax, %xmm0
1031; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm11
1032; AVX512F-NEXT:    movswl 14(%rdi), %eax
1033; AVX512F-NEXT:    vmovd %eax, %xmm0
1034; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm12
1035; AVX512F-NEXT:    movswl 12(%rdi), %eax
1036; AVX512F-NEXT:    vmovd %eax, %xmm0
1037; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm13
1038; AVX512F-NEXT:    movswl 8(%rdi), %eax
1039; AVX512F-NEXT:    vmovd %eax, %xmm0
1040; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm14
1041; AVX512F-NEXT:    movswl 10(%rdi), %eax
1042; AVX512F-NEXT:    vmovd %eax, %xmm0
1043; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm15
1044; AVX512F-NEXT:    movswl 22(%rdi), %eax
1045; AVX512F-NEXT:    vmovd %eax, %xmm0
1046; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
1047; AVX512F-NEXT:    movswl 20(%rdi), %eax
1048; AVX512F-NEXT:    vmovd %eax, %xmm1
1049; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
1050; AVX512F-NEXT:    movswl 16(%rdi), %eax
1051; AVX512F-NEXT:    vmovd %eax, %xmm2
1052; AVX512F-NEXT:    vcvtph2ps %xmm2, %xmm2
1053; AVX512F-NEXT:    movswl 18(%rdi), %eax
1054; AVX512F-NEXT:    vmovd %eax, %xmm3
1055; AVX512F-NEXT:    vcvtph2ps %xmm3, %xmm3
1056; AVX512F-NEXT:    movswl 30(%rdi), %eax
1057; AVX512F-NEXT:    vmovd %eax, %xmm4
1058; AVX512F-NEXT:    vcvtph2ps %xmm4, %xmm4
1059; AVX512F-NEXT:    movswl 28(%rdi), %eax
1060; AVX512F-NEXT:    vmovd %eax, %xmm5
1061; AVX512F-NEXT:    vcvtph2ps %xmm5, %xmm5
1062; AVX512F-NEXT:    movswl 24(%rdi), %eax
1063; AVX512F-NEXT:    vmovd %eax, %xmm6
1064; AVX512F-NEXT:    vcvtph2ps %xmm6, %xmm6
1065; AVX512F-NEXT:    movswl 26(%rdi), %eax
1066; AVX512F-NEXT:    vmovd %eax, %xmm7
1067; AVX512F-NEXT:    vcvtph2ps %xmm7, %xmm7
1068; AVX512F-NEXT:    vinsertps {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[2,3]
1069; AVX512F-NEXT:    vinsertps {{.*#+}} xmm5 = xmm6[0,1],xmm5[0],xmm6[3]
1070; AVX512F-NEXT:    vinsertps {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[0]
1071; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
1072; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
1073; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
1074; AVX512F-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
1075; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm14[0],xmm15[0],xmm14[2,3]
1076; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm13[0],xmm1[3]
1077; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm12[0]
1078; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm10[0],xmm11[0],xmm10[2,3]
1079; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
1080; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
1081; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
1082; AVX512F-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
1083; AVX512F-NEXT:    retq
1084;
1085; AVX512VL-LABEL: load_cvt_16i16_to_16f32:
1086; AVX512VL:       # BB#0:
1087; AVX512VL-NEXT:    movswl 6(%rdi), %eax
1088; AVX512VL-NEXT:    vmovd %eax, %xmm0
1089; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm8
1090; AVX512VL-NEXT:    movswl 4(%rdi), %eax
1091; AVX512VL-NEXT:    vmovd %eax, %xmm1
1092; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm9
1093; AVX512VL-NEXT:    movswl (%rdi), %eax
1094; AVX512VL-NEXT:    vmovd %eax, %xmm2
1095; AVX512VL-NEXT:    vcvtph2ps %xmm2, %xmm10
1096; AVX512VL-NEXT:    movswl 2(%rdi), %eax
1097; AVX512VL-NEXT:    vmovd %eax, %xmm3
1098; AVX512VL-NEXT:    vcvtph2ps %xmm3, %xmm11
1099; AVX512VL-NEXT:    movswl 14(%rdi), %eax
1100; AVX512VL-NEXT:    vmovd %eax, %xmm4
1101; AVX512VL-NEXT:    vcvtph2ps %xmm4, %xmm12
1102; AVX512VL-NEXT:    movswl 12(%rdi), %eax
1103; AVX512VL-NEXT:    vmovd %eax, %xmm5
1104; AVX512VL-NEXT:    vcvtph2ps %xmm5, %xmm13
1105; AVX512VL-NEXT:    movswl 8(%rdi), %eax
1106; AVX512VL-NEXT:    vmovd %eax, %xmm6
1107; AVX512VL-NEXT:    vcvtph2ps %xmm6, %xmm14
1108; AVX512VL-NEXT:    movswl 10(%rdi), %eax
1109; AVX512VL-NEXT:    vmovd %eax, %xmm7
1110; AVX512VL-NEXT:    vcvtph2ps %xmm7, %xmm15
1111; AVX512VL-NEXT:    movswl 22(%rdi), %eax
1112; AVX512VL-NEXT:    vmovd %eax, %xmm0
1113; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
1114; AVX512VL-NEXT:    movswl 20(%rdi), %eax
1115; AVX512VL-NEXT:    vmovd %eax, %xmm1
1116; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
1117; AVX512VL-NEXT:    movswl 16(%rdi), %eax
1118; AVX512VL-NEXT:    vmovd %eax, %xmm2
1119; AVX512VL-NEXT:    vcvtph2ps %xmm2, %xmm2
1120; AVX512VL-NEXT:    movswl 18(%rdi), %eax
1121; AVX512VL-NEXT:    vmovd %eax, %xmm3
1122; AVX512VL-NEXT:    vcvtph2ps %xmm3, %xmm3
1123; AVX512VL-NEXT:    movswl 30(%rdi), %eax
1124; AVX512VL-NEXT:    vmovd %eax, %xmm4
1125; AVX512VL-NEXT:    vcvtph2ps %xmm4, %xmm4
1126; AVX512VL-NEXT:    movswl 28(%rdi), %eax
1127; AVX512VL-NEXT:    vmovd %eax, %xmm5
1128; AVX512VL-NEXT:    vcvtph2ps %xmm5, %xmm5
1129; AVX512VL-NEXT:    movswl 24(%rdi), %eax
1130; AVX512VL-NEXT:    vmovd %eax, %xmm6
1131; AVX512VL-NEXT:    vcvtph2ps %xmm6, %xmm6
1132; AVX512VL-NEXT:    movswl 26(%rdi), %eax
1133; AVX512VL-NEXT:    vmovd %eax, %xmm7
1134; AVX512VL-NEXT:    vcvtph2ps %xmm7, %xmm7
1135; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[2,3]
1136; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm5 = xmm6[0,1],xmm5[0],xmm6[3]
1137; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[0]
1138; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
1139; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
1140; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
1141; AVX512VL-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
1142; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm14[0],xmm15[0],xmm14[2,3]
1143; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm13[0],xmm1[3]
1144; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm12[0]
1145; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm10[0],xmm11[0],xmm10[2,3]
1146; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
1147; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
1148; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
1149; AVX512VL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
1150; AVX512VL-NEXT:    retq
1151  %1 = load <16 x i16>, <16 x i16>* %a0
1152  %2 = bitcast <16 x i16> %1 to <16 x half>
1153  %3 = fpext <16 x half> %2 to <16 x float>
1154  ret <16 x float> %3
1155}
1156
1157;
1158; Half to Double
1159;
1160
1161define double @cvt_i16_to_f64(i16 %a0) nounwind {
1162; ALL-LABEL: cvt_i16_to_f64:
1163; ALL:       # BB#0:
1164; ALL-NEXT:    movswl %di, %eax
1165; ALL-NEXT:    vmovd %eax, %xmm0
1166; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
1167; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1168; ALL-NEXT:    retq
1169  %1 = bitcast i16 %a0 to half
1170  %2 = fpext half %1 to double
1171  ret double %2
1172}
1173
1174define <2 x double> @cvt_2i16_to_2f64(<2 x i16> %a0) nounwind {
1175; AVX1-LABEL: cvt_2i16_to_2f64:
1176; AVX1:       # BB#0:
1177; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1178; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
1179; AVX1-NEXT:    vmovd %xmm0, %eax
1180; AVX1-NEXT:    movswl %ax, %ecx
1181; AVX1-NEXT:    shrl $16, %eax
1182; AVX1-NEXT:    cwtl
1183; AVX1-NEXT:    vmovd %eax, %xmm0
1184; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
1185; AVX1-NEXT:    vmovd %ecx, %xmm1
1186; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
1187; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1188; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1189; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1190; AVX1-NEXT:    retq
1191;
1192; AVX2-LABEL: cvt_2i16_to_2f64:
1193; AVX2:       # BB#0:
1194; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1195; AVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
1196; AVX2-NEXT:    vmovd %xmm0, %eax
1197; AVX2-NEXT:    movswl %ax, %ecx
1198; AVX2-NEXT:    shrl $16, %eax
1199; AVX2-NEXT:    cwtl
1200; AVX2-NEXT:    vmovd %eax, %xmm0
1201; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
1202; AVX2-NEXT:    vmovd %ecx, %xmm1
1203; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
1204; AVX2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1205; AVX2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1206; AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1207; AVX2-NEXT:    retq
1208;
1209; AVX512F-LABEL: cvt_2i16_to_2f64:
1210; AVX512F:       # BB#0:
1211; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1212; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
1213; AVX512F-NEXT:    vmovd %xmm0, %eax
1214; AVX512F-NEXT:    movswl %ax, %ecx
1215; AVX512F-NEXT:    shrl $16, %eax
1216; AVX512F-NEXT:    cwtl
1217; AVX512F-NEXT:    vmovd %eax, %xmm0
1218; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
1219; AVX512F-NEXT:    vmovd %ecx, %xmm1
1220; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
1221; AVX512F-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1222; AVX512F-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1223; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1224; AVX512F-NEXT:    retq
1225;
1226; AVX512VL-LABEL: cvt_2i16_to_2f64:
1227; AVX512VL:       # BB#0:
1228; AVX512VL-NEXT:    vpmovqw %xmm0, -{{[0-9]+}}(%rsp)
1229; AVX512VL-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
1230; AVX512VL-NEXT:    movswl %ax, %ecx
1231; AVX512VL-NEXT:    shrl $16, %eax
1232; AVX512VL-NEXT:    cwtl
1233; AVX512VL-NEXT:    vmovd %eax, %xmm0
1234; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
1235; AVX512VL-NEXT:    vmovd %ecx, %xmm1
1236; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
1237; AVX512VL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1238; AVX512VL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1239; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1240; AVX512VL-NEXT:    retq
1241  %1 = bitcast <2 x i16> %a0 to <2 x half>
1242  %2 = fpext <2 x half> %1 to <2 x double>
1243  ret <2 x double> %2
1244}
1245
1246define <4 x double> @cvt_4i16_to_4f64(<4 x i16> %a0) nounwind {
1247; AVX1-LABEL: cvt_4i16_to_4f64:
1248; AVX1:       # BB#0:
1249; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
1250; AVX1-NEXT:    vmovq %xmm0, %rax
1251; AVX1-NEXT:    movq %rax, %rcx
1252; AVX1-NEXT:    movl %eax, %edx
1253; AVX1-NEXT:    movswl %ax, %esi
1254; AVX1-NEXT:    shrq $48, %rax
1255; AVX1-NEXT:    shrq $32, %rcx
1256; AVX1-NEXT:    shrl $16, %edx
1257; AVX1-NEXT:    movswl %dx, %edx
1258; AVX1-NEXT:    vmovd %edx, %xmm0
1259; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
1260; AVX1-NEXT:    vmovd %esi, %xmm1
1261; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
1262; AVX1-NEXT:    movswl %cx, %ecx
1263; AVX1-NEXT:    vmovd %ecx, %xmm2
1264; AVX1-NEXT:    vcvtph2ps %xmm2, %xmm2
1265; AVX1-NEXT:    cwtl
1266; AVX1-NEXT:    vmovd %eax, %xmm3
1267; AVX1-NEXT:    vcvtph2ps %xmm3, %xmm3
1268; AVX1-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1269; AVX1-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1270; AVX1-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1271; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1272; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1273; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1274; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1275; AVX1-NEXT:    retq
1276;
1277; AVX2-LABEL: cvt_4i16_to_4f64:
1278; AVX2:       # BB#0:
1279; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
1280; AVX2-NEXT:    vmovq %xmm0, %rax
1281; AVX2-NEXT:    movq %rax, %rcx
1282; AVX2-NEXT:    movl %eax, %edx
1283; AVX2-NEXT:    movswl %ax, %esi
1284; AVX2-NEXT:    shrq $48, %rax
1285; AVX2-NEXT:    shrq $32, %rcx
1286; AVX2-NEXT:    shrl $16, %edx
1287; AVX2-NEXT:    movswl %dx, %edx
1288; AVX2-NEXT:    vmovd %edx, %xmm0
1289; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
1290; AVX2-NEXT:    vmovd %esi, %xmm1
1291; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
1292; AVX2-NEXT:    movswl %cx, %ecx
1293; AVX2-NEXT:    vmovd %ecx, %xmm2
1294; AVX2-NEXT:    vcvtph2ps %xmm2, %xmm2
1295; AVX2-NEXT:    cwtl
1296; AVX2-NEXT:    vmovd %eax, %xmm3
1297; AVX2-NEXT:    vcvtph2ps %xmm3, %xmm3
1298; AVX2-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1299; AVX2-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1300; AVX2-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1301; AVX2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1302; AVX2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1303; AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1304; AVX2-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1305; AVX2-NEXT:    retq
1306;
1307; AVX512F-LABEL: cvt_4i16_to_4f64:
1308; AVX512F:       # BB#0:
1309; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
1310; AVX512F-NEXT:    vmovq %xmm0, %rax
1311; AVX512F-NEXT:    movq %rax, %rcx
1312; AVX512F-NEXT:    movl %eax, %edx
1313; AVX512F-NEXT:    movswl %ax, %esi
1314; AVX512F-NEXT:    shrq $48, %rax
1315; AVX512F-NEXT:    shrq $32, %rcx
1316; AVX512F-NEXT:    shrl $16, %edx
1317; AVX512F-NEXT:    movswl %dx, %edx
1318; AVX512F-NEXT:    vmovd %edx, %xmm0
1319; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
1320; AVX512F-NEXT:    vmovd %esi, %xmm1
1321; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
1322; AVX512F-NEXT:    movswl %cx, %ecx
1323; AVX512F-NEXT:    vmovd %ecx, %xmm2
1324; AVX512F-NEXT:    vcvtph2ps %xmm2, %xmm2
1325; AVX512F-NEXT:    cwtl
1326; AVX512F-NEXT:    vmovd %eax, %xmm3
1327; AVX512F-NEXT:    vcvtph2ps %xmm3, %xmm3
1328; AVX512F-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1329; AVX512F-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1330; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1331; AVX512F-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1332; AVX512F-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1333; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1334; AVX512F-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1335; AVX512F-NEXT:    retq
1336;
1337; AVX512VL-LABEL: cvt_4i16_to_4f64:
1338; AVX512VL:       # BB#0:
1339; AVX512VL-NEXT:    vpmovdw %xmm0, -{{[0-9]+}}(%rsp)
1340; AVX512VL-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
1341; AVX512VL-NEXT:    movq %rax, %rcx
1342; AVX512VL-NEXT:    movl %eax, %edx
1343; AVX512VL-NEXT:    movswl %ax, %esi
1344; AVX512VL-NEXT:    shrq $48, %rax
1345; AVX512VL-NEXT:    shrq $32, %rcx
1346; AVX512VL-NEXT:    shrl $16, %edx
1347; AVX512VL-NEXT:    movswl %dx, %edx
1348; AVX512VL-NEXT:    vmovd %edx, %xmm0
1349; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
1350; AVX512VL-NEXT:    vmovd %esi, %xmm1
1351; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
1352; AVX512VL-NEXT:    movswl %cx, %ecx
1353; AVX512VL-NEXT:    vmovd %ecx, %xmm2
1354; AVX512VL-NEXT:    vcvtph2ps %xmm2, %xmm2
1355; AVX512VL-NEXT:    cwtl
1356; AVX512VL-NEXT:    vmovd %eax, %xmm3
1357; AVX512VL-NEXT:    vcvtph2ps %xmm3, %xmm3
1358; AVX512VL-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1359; AVX512VL-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1360; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1361; AVX512VL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1362; AVX512VL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1363; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1364; AVX512VL-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1365; AVX512VL-NEXT:    retq
1366  %1 = bitcast <4 x i16> %a0 to <4 x half>
1367  %2 = fpext <4 x half> %1 to <4 x double>
1368  ret <4 x double> %2
1369}
1370
1371define <2 x double> @cvt_8i16_to_2f64(<8 x i16> %a0) nounwind {
1372; AVX1-LABEL: cvt_8i16_to_2f64:
1373; AVX1:       # BB#0:
1374; AVX1-NEXT:    vmovd %xmm0, %eax
1375; AVX1-NEXT:    movswl %ax, %ecx
1376; AVX1-NEXT:    shrl $16, %eax
1377; AVX1-NEXT:    cwtl
1378; AVX1-NEXT:    vmovd %eax, %xmm0
1379; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
1380; AVX1-NEXT:    vmovd %ecx, %xmm1
1381; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
1382; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1383; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1384; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1385; AVX1-NEXT:    retq
1386;
1387; AVX2-LABEL: cvt_8i16_to_2f64:
1388; AVX2:       # BB#0:
1389; AVX2-NEXT:    vmovd %xmm0, %eax
1390; AVX2-NEXT:    movswl %ax, %ecx
1391; AVX2-NEXT:    shrl $16, %eax
1392; AVX2-NEXT:    cwtl
1393; AVX2-NEXT:    vmovd %eax, %xmm0
1394; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
1395; AVX2-NEXT:    vmovd %ecx, %xmm1
1396; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
1397; AVX2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1398; AVX2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1399; AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1400; AVX2-NEXT:    retq
1401;
1402; AVX512F-LABEL: cvt_8i16_to_2f64:
1403; AVX512F:       # BB#0:
1404; AVX512F-NEXT:    vmovd %xmm0, %eax
1405; AVX512F-NEXT:    movswl %ax, %ecx
1406; AVX512F-NEXT:    shrl $16, %eax
1407; AVX512F-NEXT:    cwtl
1408; AVX512F-NEXT:    vmovd %eax, %xmm0
1409; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
1410; AVX512F-NEXT:    vmovd %ecx, %xmm1
1411; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
1412; AVX512F-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1413; AVX512F-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1414; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1415; AVX512F-NEXT:    retq
1416;
1417; AVX512VL-LABEL: cvt_8i16_to_2f64:
1418; AVX512VL:       # BB#0:
1419; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
1420; AVX512VL-NEXT:    vpmovqw %xmm0, -{{[0-9]+}}(%rsp)
1421; AVX512VL-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
1422; AVX512VL-NEXT:    movswl %ax, %ecx
1423; AVX512VL-NEXT:    shrl $16, %eax
1424; AVX512VL-NEXT:    cwtl
1425; AVX512VL-NEXT:    vmovd %eax, %xmm0
1426; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
1427; AVX512VL-NEXT:    vmovd %ecx, %xmm1
1428; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
1429; AVX512VL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1430; AVX512VL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1431; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1432; AVX512VL-NEXT:    retq
1433  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
1434  %2 = bitcast <2 x i16> %1 to <2 x half>
1435  %3 = fpext <2 x half> %2 to <2 x double>
1436  ret <2 x double> %3
1437}
1438
1439define <4 x double> @cvt_8i16_to_4f64(<8 x i16> %a0) nounwind {
1440; AVX1-LABEL: cvt_8i16_to_4f64:
1441; AVX1:       # BB#0:
1442; AVX1-NEXT:    vmovq %xmm0, %rax
1443; AVX1-NEXT:    movq %rax, %rcx
1444; AVX1-NEXT:    movl %eax, %edx
1445; AVX1-NEXT:    movswl %ax, %esi
1446; AVX1-NEXT:    shrq $48, %rax
1447; AVX1-NEXT:    shrq $32, %rcx
1448; AVX1-NEXT:    shrl $16, %edx
1449; AVX1-NEXT:    movswl %dx, %edx
1450; AVX1-NEXT:    vmovd %edx, %xmm0
1451; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
1452; AVX1-NEXT:    vmovd %esi, %xmm1
1453; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
1454; AVX1-NEXT:    movswl %cx, %ecx
1455; AVX1-NEXT:    vmovd %ecx, %xmm2
1456; AVX1-NEXT:    vcvtph2ps %xmm2, %xmm2
1457; AVX1-NEXT:    cwtl
1458; AVX1-NEXT:    vmovd %eax, %xmm3
1459; AVX1-NEXT:    vcvtph2ps %xmm3, %xmm3
1460; AVX1-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1461; AVX1-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1462; AVX1-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1463; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1464; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1465; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1466; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1467; AVX1-NEXT:    retq
1468;
1469; AVX2-LABEL: cvt_8i16_to_4f64:
1470; AVX2:       # BB#0:
1471; AVX2-NEXT:    vmovq %xmm0, %rax
1472; AVX2-NEXT:    movq %rax, %rcx
1473; AVX2-NEXT:    movl %eax, %edx
1474; AVX2-NEXT:    movswl %ax, %esi
1475; AVX2-NEXT:    shrq $48, %rax
1476; AVX2-NEXT:    shrq $32, %rcx
1477; AVX2-NEXT:    shrl $16, %edx
1478; AVX2-NEXT:    movswl %dx, %edx
1479; AVX2-NEXT:    vmovd %edx, %xmm0
1480; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
1481; AVX2-NEXT:    vmovd %esi, %xmm1
1482; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
1483; AVX2-NEXT:    movswl %cx, %ecx
1484; AVX2-NEXT:    vmovd %ecx, %xmm2
1485; AVX2-NEXT:    vcvtph2ps %xmm2, %xmm2
1486; AVX2-NEXT:    cwtl
1487; AVX2-NEXT:    vmovd %eax, %xmm3
1488; AVX2-NEXT:    vcvtph2ps %xmm3, %xmm3
1489; AVX2-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1490; AVX2-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1491; AVX2-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1492; AVX2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1493; AVX2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1494; AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1495; AVX2-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1496; AVX2-NEXT:    retq
1497;
1498; AVX512F-LABEL: cvt_8i16_to_4f64:
1499; AVX512F:       # BB#0:
1500; AVX512F-NEXT:    vmovq %xmm0, %rax
1501; AVX512F-NEXT:    movq %rax, %rcx
1502; AVX512F-NEXT:    movl %eax, %edx
1503; AVX512F-NEXT:    movswl %ax, %esi
1504; AVX512F-NEXT:    shrq $48, %rax
1505; AVX512F-NEXT:    shrq $32, %rcx
1506; AVX512F-NEXT:    shrl $16, %edx
1507; AVX512F-NEXT:    movswl %dx, %edx
1508; AVX512F-NEXT:    vmovd %edx, %xmm0
1509; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
1510; AVX512F-NEXT:    vmovd %esi, %xmm1
1511; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
1512; AVX512F-NEXT:    movswl %cx, %ecx
1513; AVX512F-NEXT:    vmovd %ecx, %xmm2
1514; AVX512F-NEXT:    vcvtph2ps %xmm2, %xmm2
1515; AVX512F-NEXT:    cwtl
1516; AVX512F-NEXT:    vmovd %eax, %xmm3
1517; AVX512F-NEXT:    vcvtph2ps %xmm3, %xmm3
1518; AVX512F-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1519; AVX512F-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1520; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1521; AVX512F-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1522; AVX512F-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1523; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1524; AVX512F-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1525; AVX512F-NEXT:    retq
1526;
1527; AVX512VL-LABEL: cvt_8i16_to_4f64:
1528; AVX512VL:       # BB#0:
1529; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
1530; AVX512VL-NEXT:    vpmovdw %xmm0, -{{[0-9]+}}(%rsp)
1531; AVX512VL-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
1532; AVX512VL-NEXT:    movq %rax, %rcx
1533; AVX512VL-NEXT:    movl %eax, %edx
1534; AVX512VL-NEXT:    movswl %ax, %esi
1535; AVX512VL-NEXT:    shrq $48, %rax
1536; AVX512VL-NEXT:    shrq $32, %rcx
1537; AVX512VL-NEXT:    shrl $16, %edx
1538; AVX512VL-NEXT:    movswl %dx, %edx
1539; AVX512VL-NEXT:    vmovd %edx, %xmm0
1540; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
1541; AVX512VL-NEXT:    vmovd %esi, %xmm1
1542; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
1543; AVX512VL-NEXT:    movswl %cx, %ecx
1544; AVX512VL-NEXT:    vmovd %ecx, %xmm2
1545; AVX512VL-NEXT:    vcvtph2ps %xmm2, %xmm2
1546; AVX512VL-NEXT:    cwtl
1547; AVX512VL-NEXT:    vmovd %eax, %xmm3
1548; AVX512VL-NEXT:    vcvtph2ps %xmm3, %xmm3
1549; AVX512VL-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1550; AVX512VL-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1551; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1552; AVX512VL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1553; AVX512VL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1554; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1555; AVX512VL-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1556; AVX512VL-NEXT:    retq
1557  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1558  %2 = bitcast <4 x i16> %1 to <4 x half>
1559  %3 = fpext <4 x half> %2 to <4 x double>
1560  ret <4 x double> %3
1561}
1562
1563define <8 x double> @cvt_8i16_to_8f64(<8 x i16> %a0) nounwind {
1564; AVX1-LABEL: cvt_8i16_to_8f64:
1565; AVX1:       # BB#0:
1566; AVX1-NEXT:    vmovq %xmm0, %rdx
1567; AVX1-NEXT:    movq %rdx, %r9
1568; AVX1-NEXT:    movl %edx, %r10d
1569; AVX1-NEXT:    movswl %dx, %r8d
1570; AVX1-NEXT:    shrq $48, %rdx
1571; AVX1-NEXT:    shrq $32, %r9
1572; AVX1-NEXT:    shrl $16, %r10d
1573; AVX1-NEXT:    vpextrq $1, %xmm0, %rdi
1574; AVX1-NEXT:    movq %rdi, %rsi
1575; AVX1-NEXT:    movl %edi, %eax
1576; AVX1-NEXT:    movswl %di, %ecx
1577; AVX1-NEXT:    shrq $48, %rdi
1578; AVX1-NEXT:    shrq $32, %rsi
1579; AVX1-NEXT:    shrl $16, %eax
1580; AVX1-NEXT:    cwtl
1581; AVX1-NEXT:    vmovd %eax, %xmm0
1582; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm1
1583; AVX1-NEXT:    vmovd %ecx, %xmm0
1584; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm2
1585; AVX1-NEXT:    movswl %si, %eax
1586; AVX1-NEXT:    vmovd %eax, %xmm0
1587; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm3
1588; AVX1-NEXT:    movswl %di, %eax
1589; AVX1-NEXT:    vmovd %eax, %xmm0
1590; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm4
1591; AVX1-NEXT:    movswl %r10w, %eax
1592; AVX1-NEXT:    vmovd %eax, %xmm0
1593; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
1594; AVX1-NEXT:    vmovd %r8d, %xmm5
1595; AVX1-NEXT:    vcvtph2ps %xmm5, %xmm5
1596; AVX1-NEXT:    movswl %r9w, %eax
1597; AVX1-NEXT:    vmovd %eax, %xmm6
1598; AVX1-NEXT:    vcvtph2ps %xmm6, %xmm6
1599; AVX1-NEXT:    movswl %dx, %eax
1600; AVX1-NEXT:    vmovd %eax, %xmm7
1601; AVX1-NEXT:    vcvtph2ps %xmm7, %xmm7
1602; AVX1-NEXT:    vcvtss2sd %xmm7, %xmm7, %xmm7
1603; AVX1-NEXT:    vcvtss2sd %xmm6, %xmm6, %xmm6
1604; AVX1-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
1605; AVX1-NEXT:    vcvtss2sd %xmm5, %xmm5, %xmm5
1606; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1607; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm5[0],xmm0[0]
1608; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
1609; AVX1-NEXT:    vcvtss2sd %xmm4, %xmm4, %xmm4
1610; AVX1-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1611; AVX1-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
1612; AVX1-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1613; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1614; AVX1-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0]
1615; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
1616; AVX1-NEXT:    retq
1617;
1618; AVX2-LABEL: cvt_8i16_to_8f64:
1619; AVX2:       # BB#0:
1620; AVX2-NEXT:    vmovq %xmm0, %rdx
1621; AVX2-NEXT:    movq %rdx, %r9
1622; AVX2-NEXT:    movl %edx, %r10d
1623; AVX2-NEXT:    movswl %dx, %r8d
1624; AVX2-NEXT:    shrq $48, %rdx
1625; AVX2-NEXT:    shrq $32, %r9
1626; AVX2-NEXT:    shrl $16, %r10d
1627; AVX2-NEXT:    vpextrq $1, %xmm0, %rdi
1628; AVX2-NEXT:    movq %rdi, %rsi
1629; AVX2-NEXT:    movl %edi, %eax
1630; AVX2-NEXT:    movswl %di, %ecx
1631; AVX2-NEXT:    shrq $48, %rdi
1632; AVX2-NEXT:    shrq $32, %rsi
1633; AVX2-NEXT:    shrl $16, %eax
1634; AVX2-NEXT:    cwtl
1635; AVX2-NEXT:    vmovd %eax, %xmm0
1636; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm1
1637; AVX2-NEXT:    vmovd %ecx, %xmm0
1638; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm2
1639; AVX2-NEXT:    movswl %si, %eax
1640; AVX2-NEXT:    vmovd %eax, %xmm0
1641; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm3
1642; AVX2-NEXT:    movswl %di, %eax
1643; AVX2-NEXT:    vmovd %eax, %xmm0
1644; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm4
1645; AVX2-NEXT:    movswl %r10w, %eax
1646; AVX2-NEXT:    vmovd %eax, %xmm0
1647; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
1648; AVX2-NEXT:    vmovd %r8d, %xmm5
1649; AVX2-NEXT:    vcvtph2ps %xmm5, %xmm5
1650; AVX2-NEXT:    movswl %r9w, %eax
1651; AVX2-NEXT:    vmovd %eax, %xmm6
1652; AVX2-NEXT:    vcvtph2ps %xmm6, %xmm6
1653; AVX2-NEXT:    movswl %dx, %eax
1654; AVX2-NEXT:    vmovd %eax, %xmm7
1655; AVX2-NEXT:    vcvtph2ps %xmm7, %xmm7
1656; AVX2-NEXT:    vcvtss2sd %xmm7, %xmm7, %xmm7
1657; AVX2-NEXT:    vcvtss2sd %xmm6, %xmm6, %xmm6
1658; AVX2-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
1659; AVX2-NEXT:    vcvtss2sd %xmm5, %xmm5, %xmm5
1660; AVX2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1661; AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm5[0],xmm0[0]
1662; AVX2-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
1663; AVX2-NEXT:    vcvtss2sd %xmm4, %xmm4, %xmm4
1664; AVX2-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1665; AVX2-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
1666; AVX2-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1667; AVX2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1668; AVX2-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0]
1669; AVX2-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
1670; AVX2-NEXT:    retq
1671;
1672; AVX512-LABEL: cvt_8i16_to_8f64:
1673; AVX512:       # BB#0:
1674; AVX512-NEXT:    vpextrq $1, %xmm0, %rdx
1675; AVX512-NEXT:    movq %rdx, %r9
1676; AVX512-NEXT:    movl %edx, %r10d
1677; AVX512-NEXT:    movswl %dx, %r8d
1678; AVX512-NEXT:    shrq $48, %rdx
1679; AVX512-NEXT:    shrq $32, %r9
1680; AVX512-NEXT:    shrl $16, %r10d
1681; AVX512-NEXT:    vmovq %xmm0, %rdi
1682; AVX512-NEXT:    movq %rdi, %rsi
1683; AVX512-NEXT:    movl %edi, %eax
1684; AVX512-NEXT:    movswl %di, %ecx
1685; AVX512-NEXT:    shrq $48, %rdi
1686; AVX512-NEXT:    shrq $32, %rsi
1687; AVX512-NEXT:    shrl $16, %eax
1688; AVX512-NEXT:    cwtl
1689; AVX512-NEXT:    vmovd %eax, %xmm0
1690; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
1691; AVX512-NEXT:    vmovd %ecx, %xmm1
1692; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
1693; AVX512-NEXT:    movswl %si, %eax
1694; AVX512-NEXT:    vmovd %eax, %xmm2
1695; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
1696; AVX512-NEXT:    movswl %di, %eax
1697; AVX512-NEXT:    vmovd %eax, %xmm3
1698; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
1699; AVX512-NEXT:    movswl %r10w, %eax
1700; AVX512-NEXT:    vmovd %eax, %xmm4
1701; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm4
1702; AVX512-NEXT:    vmovd %r8d, %xmm5
1703; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
1704; AVX512-NEXT:    movswl %r9w, %eax
1705; AVX512-NEXT:    vmovd %eax, %xmm6
1706; AVX512-NEXT:    vcvtph2ps %xmm6, %xmm6
1707; AVX512-NEXT:    movswl %dx, %eax
1708; AVX512-NEXT:    vmovd %eax, %xmm7
1709; AVX512-NEXT:    vcvtph2ps %xmm7, %xmm7
1710; AVX512-NEXT:    vcvtss2sd %xmm7, %xmm7, %xmm7
1711; AVX512-NEXT:    vcvtss2sd %xmm6, %xmm6, %xmm6
1712; AVX512-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
1713; AVX512-NEXT:    vcvtss2sd %xmm5, %xmm5, %xmm5
1714; AVX512-NEXT:    vcvtss2sd %xmm4, %xmm4, %xmm4
1715; AVX512-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm5[0],xmm4[0]
1716; AVX512-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm4
1717; AVX512-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1718; AVX512-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1719; AVX512-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1720; AVX512-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1721; AVX512-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1722; AVX512-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1723; AVX512-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1724; AVX512-NEXT:    vinsertf64x4 $1, %ymm4, %zmm0, %zmm0
1725; AVX512-NEXT:    retq
1726  %1 = bitcast <8 x i16> %a0 to <8 x half>
1727  %2 = fpext <8 x half> %1 to <8 x double>
1728  ret <8 x double> %2
1729}
1730
1731;
1732; Half to Double (Load)
1733;
1734
1735define double @load_cvt_i16_to_f64(i16* %a0) nounwind {
1736; ALL-LABEL: load_cvt_i16_to_f64:
1737; ALL:       # BB#0:
1738; ALL-NEXT:    movswl (%rdi), %eax
1739; ALL-NEXT:    vmovd %eax, %xmm0
1740; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
1741; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1742; ALL-NEXT:    retq
1743  %1 = load i16, i16* %a0
1744  %2 = bitcast i16 %1 to half
1745  %3 = fpext half %2 to double
1746  ret double %3
1747}
1748
1749define <2 x double> @load_cvt_2i16_to_2f64(<2 x i16>* %a0) nounwind {
1750; ALL-LABEL: load_cvt_2i16_to_2f64:
1751; ALL:       # BB#0:
1752; ALL-NEXT:    movswl (%rdi), %eax
1753; ALL-NEXT:    vmovd %eax, %xmm0
1754; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
1755; ALL-NEXT:    movswl 2(%rdi), %eax
1756; ALL-NEXT:    vmovd %eax, %xmm1
1757; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
1758; ALL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1759; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1760; ALL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1761; ALL-NEXT:    retq
1762  %1 = load <2 x i16>, <2 x i16>* %a0
1763  %2 = bitcast <2 x i16> %1 to <2 x half>
1764  %3 = fpext <2 x half> %2 to <2 x double>
1765  ret <2 x double> %3
1766}
1767
1768define <4 x double> @load_cvt_4i16_to_4f64(<4 x i16>* %a0) nounwind {
1769; ALL-LABEL: load_cvt_4i16_to_4f64:
1770; ALL:       # BB#0:
1771; ALL-NEXT:    movswl (%rdi), %eax
1772; ALL-NEXT:    vmovd %eax, %xmm0
1773; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
1774; ALL-NEXT:    movswl 2(%rdi), %eax
1775; ALL-NEXT:    vmovd %eax, %xmm1
1776; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
1777; ALL-NEXT:    movswl 4(%rdi), %eax
1778; ALL-NEXT:    vmovd %eax, %xmm2
1779; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
1780; ALL-NEXT:    movswl 6(%rdi), %eax
1781; ALL-NEXT:    vmovd %eax, %xmm3
1782; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
1783; ALL-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1784; ALL-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1785; ALL-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1786; ALL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1787; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1788; ALL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1789; ALL-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1790; ALL-NEXT:    retq
1791  %1 = load <4 x i16>, <4 x i16>* %a0
1792  %2 = bitcast <4 x i16> %1 to <4 x half>
1793  %3 = fpext <4 x half> %2 to <4 x double>
1794  ret <4 x double> %3
1795}
1796
1797define <4 x double> @load_cvt_8i16_to_4f64(<8 x i16>* %a0) nounwind {
1798; AVX1-LABEL: load_cvt_8i16_to_4f64:
1799; AVX1:       # BB#0:
1800; AVX1-NEXT:    movq (%rdi), %rax
1801; AVX1-NEXT:    movq %rax, %rcx
1802; AVX1-NEXT:    movl %eax, %edx
1803; AVX1-NEXT:    movswl %ax, %esi
1804; AVX1-NEXT:    shrq $48, %rax
1805; AVX1-NEXT:    shrq $32, %rcx
1806; AVX1-NEXT:    shrl $16, %edx
1807; AVX1-NEXT:    movswl %dx, %edx
1808; AVX1-NEXT:    vmovd %edx, %xmm0
1809; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
1810; AVX1-NEXT:    vmovd %esi, %xmm1
1811; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
1812; AVX1-NEXT:    movswl %cx, %ecx
1813; AVX1-NEXT:    vmovd %ecx, %xmm2
1814; AVX1-NEXT:    vcvtph2ps %xmm2, %xmm2
1815; AVX1-NEXT:    cwtl
1816; AVX1-NEXT:    vmovd %eax, %xmm3
1817; AVX1-NEXT:    vcvtph2ps %xmm3, %xmm3
1818; AVX1-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1819; AVX1-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1820; AVX1-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1821; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1822; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1823; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1824; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1825; AVX1-NEXT:    retq
1826;
1827; AVX2-LABEL: load_cvt_8i16_to_4f64:
1828; AVX2:       # BB#0:
1829; AVX2-NEXT:    movq (%rdi), %rax
1830; AVX2-NEXT:    movq %rax, %rcx
1831; AVX2-NEXT:    movl %eax, %edx
1832; AVX2-NEXT:    movswl %ax, %esi
1833; AVX2-NEXT:    shrq $48, %rax
1834; AVX2-NEXT:    shrq $32, %rcx
1835; AVX2-NEXT:    shrl $16, %edx
1836; AVX2-NEXT:    movswl %dx, %edx
1837; AVX2-NEXT:    vmovd %edx, %xmm0
1838; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
1839; AVX2-NEXT:    vmovd %esi, %xmm1
1840; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
1841; AVX2-NEXT:    movswl %cx, %ecx
1842; AVX2-NEXT:    vmovd %ecx, %xmm2
1843; AVX2-NEXT:    vcvtph2ps %xmm2, %xmm2
1844; AVX2-NEXT:    cwtl
1845; AVX2-NEXT:    vmovd %eax, %xmm3
1846; AVX2-NEXT:    vcvtph2ps %xmm3, %xmm3
1847; AVX2-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1848; AVX2-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1849; AVX2-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1850; AVX2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1851; AVX2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1852; AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1853; AVX2-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1854; AVX2-NEXT:    retq
1855;
1856; AVX512F-LABEL: load_cvt_8i16_to_4f64:
1857; AVX512F:       # BB#0:
1858; AVX512F-NEXT:    movq (%rdi), %rax
1859; AVX512F-NEXT:    movq %rax, %rcx
1860; AVX512F-NEXT:    movl %eax, %edx
1861; AVX512F-NEXT:    movswl %ax, %esi
1862; AVX512F-NEXT:    shrq $48, %rax
1863; AVX512F-NEXT:    shrq $32, %rcx
1864; AVX512F-NEXT:    shrl $16, %edx
1865; AVX512F-NEXT:    movswl %dx, %edx
1866; AVX512F-NEXT:    vmovd %edx, %xmm0
1867; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
1868; AVX512F-NEXT:    vmovd %esi, %xmm1
1869; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
1870; AVX512F-NEXT:    movswl %cx, %ecx
1871; AVX512F-NEXT:    vmovd %ecx, %xmm2
1872; AVX512F-NEXT:    vcvtph2ps %xmm2, %xmm2
1873; AVX512F-NEXT:    cwtl
1874; AVX512F-NEXT:    vmovd %eax, %xmm3
1875; AVX512F-NEXT:    vcvtph2ps %xmm3, %xmm3
1876; AVX512F-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1877; AVX512F-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1878; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1879; AVX512F-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1880; AVX512F-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1881; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1882; AVX512F-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1883; AVX512F-NEXT:    retq
1884;
1885; AVX512VL-LABEL: load_cvt_8i16_to_4f64:
1886; AVX512VL:       # BB#0:
1887; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1888; AVX512VL-NEXT:    vpmovdw %xmm0, -{{[0-9]+}}(%rsp)
1889; AVX512VL-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
1890; AVX512VL-NEXT:    movq %rax, %rcx
1891; AVX512VL-NEXT:    movl %eax, %edx
1892; AVX512VL-NEXT:    movswl %ax, %esi
1893; AVX512VL-NEXT:    shrq $48, %rax
1894; AVX512VL-NEXT:    shrq $32, %rcx
1895; AVX512VL-NEXT:    shrl $16, %edx
1896; AVX512VL-NEXT:    movswl %dx, %edx
1897; AVX512VL-NEXT:    vmovd %edx, %xmm0
1898; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
1899; AVX512VL-NEXT:    vmovd %esi, %xmm1
1900; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
1901; AVX512VL-NEXT:    movswl %cx, %ecx
1902; AVX512VL-NEXT:    vmovd %ecx, %xmm2
1903; AVX512VL-NEXT:    vcvtph2ps %xmm2, %xmm2
1904; AVX512VL-NEXT:    cwtl
1905; AVX512VL-NEXT:    vmovd %eax, %xmm3
1906; AVX512VL-NEXT:    vcvtph2ps %xmm3, %xmm3
1907; AVX512VL-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1908; AVX512VL-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1909; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1910; AVX512VL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1911; AVX512VL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1912; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1913; AVX512VL-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1914; AVX512VL-NEXT:    retq
1915  %1 = load <8 x i16>, <8 x i16>* %a0
1916  %2 = shufflevector <8 x i16> %1, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1917  %3 = bitcast <4 x i16> %2 to <4 x half>
1918  %4 = fpext <4 x half> %3 to <4 x double>
1919  ret <4 x double> %4
1920}
1921
1922define <8 x double> @load_cvt_8i16_to_8f64(<8 x i16>* %a0) nounwind {
1923; AVX1-LABEL: load_cvt_8i16_to_8f64:
1924; AVX1:       # BB#0:
1925; AVX1-NEXT:    movswl 8(%rdi), %eax
1926; AVX1-NEXT:    vmovd %eax, %xmm0
1927; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm1
1928; AVX1-NEXT:    movswl 10(%rdi), %eax
1929; AVX1-NEXT:    vmovd %eax, %xmm0
1930; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm2
1931; AVX1-NEXT:    movswl 12(%rdi), %eax
1932; AVX1-NEXT:    vmovd %eax, %xmm0
1933; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm3
1934; AVX1-NEXT:    movswl 14(%rdi), %eax
1935; AVX1-NEXT:    vmovd %eax, %xmm0
1936; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm4
1937; AVX1-NEXT:    movswl (%rdi), %eax
1938; AVX1-NEXT:    vmovd %eax, %xmm0
1939; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
1940; AVX1-NEXT:    movswl 2(%rdi), %eax
1941; AVX1-NEXT:    vmovd %eax, %xmm5
1942; AVX1-NEXT:    vcvtph2ps %xmm5, %xmm5
1943; AVX1-NEXT:    movswl 4(%rdi), %eax
1944; AVX1-NEXT:    vmovd %eax, %xmm6
1945; AVX1-NEXT:    vcvtph2ps %xmm6, %xmm6
1946; AVX1-NEXT:    movswl 6(%rdi), %eax
1947; AVX1-NEXT:    vmovd %eax, %xmm7
1948; AVX1-NEXT:    vcvtph2ps %xmm7, %xmm7
1949; AVX1-NEXT:    vcvtss2sd %xmm7, %xmm7, %xmm7
1950; AVX1-NEXT:    vcvtss2sd %xmm6, %xmm6, %xmm6
1951; AVX1-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
1952; AVX1-NEXT:    vcvtss2sd %xmm5, %xmm5, %xmm5
1953; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1954; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm5[0]
1955; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
1956; AVX1-NEXT:    vcvtss2sd %xmm4, %xmm4, %xmm4
1957; AVX1-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1958; AVX1-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
1959; AVX1-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1960; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1961; AVX1-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
1962; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
1963; AVX1-NEXT:    retq
1964;
1965; AVX2-LABEL: load_cvt_8i16_to_8f64:
1966; AVX2:       # BB#0:
1967; AVX2-NEXT:    movswl 8(%rdi), %eax
1968; AVX2-NEXT:    vmovd %eax, %xmm0
1969; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm1
1970; AVX2-NEXT:    movswl 10(%rdi), %eax
1971; AVX2-NEXT:    vmovd %eax, %xmm0
1972; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm2
1973; AVX2-NEXT:    movswl 12(%rdi), %eax
1974; AVX2-NEXT:    vmovd %eax, %xmm0
1975; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm3
1976; AVX2-NEXT:    movswl 14(%rdi), %eax
1977; AVX2-NEXT:    vmovd %eax, %xmm0
1978; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm4
1979; AVX2-NEXT:    movswl (%rdi), %eax
1980; AVX2-NEXT:    vmovd %eax, %xmm0
1981; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
1982; AVX2-NEXT:    movswl 2(%rdi), %eax
1983; AVX2-NEXT:    vmovd %eax, %xmm5
1984; AVX2-NEXT:    vcvtph2ps %xmm5, %xmm5
1985; AVX2-NEXT:    movswl 4(%rdi), %eax
1986; AVX2-NEXT:    vmovd %eax, %xmm6
1987; AVX2-NEXT:    vcvtph2ps %xmm6, %xmm6
1988; AVX2-NEXT:    movswl 6(%rdi), %eax
1989; AVX2-NEXT:    vmovd %eax, %xmm7
1990; AVX2-NEXT:    vcvtph2ps %xmm7, %xmm7
1991; AVX2-NEXT:    vcvtss2sd %xmm7, %xmm7, %xmm7
1992; AVX2-NEXT:    vcvtss2sd %xmm6, %xmm6, %xmm6
1993; AVX2-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
1994; AVX2-NEXT:    vcvtss2sd %xmm5, %xmm5, %xmm5
1995; AVX2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1996; AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm5[0]
1997; AVX2-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
1998; AVX2-NEXT:    vcvtss2sd %xmm4, %xmm4, %xmm4
1999; AVX2-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
2000; AVX2-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
2001; AVX2-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
2002; AVX2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
2003; AVX2-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
2004; AVX2-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
2005; AVX2-NEXT:    retq
2006;
2007; AVX512-LABEL: load_cvt_8i16_to_8f64:
2008; AVX512:       # BB#0:
2009; AVX512-NEXT:    movswl (%rdi), %eax
2010; AVX512-NEXT:    vmovd %eax, %xmm0
2011; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
2012; AVX512-NEXT:    movswl 2(%rdi), %eax
2013; AVX512-NEXT:    vmovd %eax, %xmm1
2014; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
2015; AVX512-NEXT:    movswl 4(%rdi), %eax
2016; AVX512-NEXT:    vmovd %eax, %xmm2
2017; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
2018; AVX512-NEXT:    movswl 6(%rdi), %eax
2019; AVX512-NEXT:    vmovd %eax, %xmm3
2020; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
2021; AVX512-NEXT:    movswl 8(%rdi), %eax
2022; AVX512-NEXT:    vmovd %eax, %xmm4
2023; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm4
2024; AVX512-NEXT:    movswl 10(%rdi), %eax
2025; AVX512-NEXT:    vmovd %eax, %xmm5
2026; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
2027; AVX512-NEXT:    movswl 12(%rdi), %eax
2028; AVX512-NEXT:    vmovd %eax, %xmm6
2029; AVX512-NEXT:    vcvtph2ps %xmm6, %xmm6
2030; AVX512-NEXT:    movswl 14(%rdi), %eax
2031; AVX512-NEXT:    vmovd %eax, %xmm7
2032; AVX512-NEXT:    vcvtph2ps %xmm7, %xmm7
2033; AVX512-NEXT:    vcvtss2sd %xmm7, %xmm7, %xmm7
2034; AVX512-NEXT:    vcvtss2sd %xmm6, %xmm6, %xmm6
2035; AVX512-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
2036; AVX512-NEXT:    vcvtss2sd %xmm5, %xmm5, %xmm5
2037; AVX512-NEXT:    vcvtss2sd %xmm4, %xmm4, %xmm4
2038; AVX512-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm4[0],xmm5[0]
2039; AVX512-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm4
2040; AVX512-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
2041; AVX512-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
2042; AVX512-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
2043; AVX512-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
2044; AVX512-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
2045; AVX512-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2046; AVX512-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
2047; AVX512-NEXT:    vinsertf64x4 $1, %ymm4, %zmm0, %zmm0
2048; AVX512-NEXT:    retq
2049  %1 = load <8 x i16>, <8 x i16>* %a0
2050  %2 = bitcast <8 x i16> %1 to <8 x half>
2051  %3 = fpext <8 x half> %2 to <8 x double>
2052  ret <8 x double> %3
2053}
2054
2055;
2056; Float to Half
2057;
2058
2059define i16 @cvt_f32_to_i16(float %a0) nounwind {
2060; ALL-LABEL: cvt_f32_to_i16:
2061; ALL:       # BB#0:
2062; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2063; ALL-NEXT:    vmovd %xmm0, %eax
2064; ALL-NEXT:    # kill: %AX<def> %AX<kill> %EAX<kill>
2065; ALL-NEXT:    retq
2066  %1 = fptrunc float %a0 to half
2067  %2 = bitcast half %1 to i16
2068  ret i16 %2
2069}
2070
2071define <4 x i16> @cvt_4f32_to_4i16(<4 x float> %a0) nounwind {
2072; ALL-LABEL: cvt_4f32_to_4i16:
2073; ALL:       # BB#0:
2074; ALL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2075; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2076; ALL-NEXT:    vmovd %xmm1, %eax
2077; ALL-NEXT:    shll $16, %eax
2078; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2079; ALL-NEXT:    vmovd %xmm1, %ecx
2080; ALL-NEXT:    movzwl %cx, %ecx
2081; ALL-NEXT:    orl %eax, %ecx
2082; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2083; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2084; ALL-NEXT:    vmovd %xmm1, %eax
2085; ALL-NEXT:    shll $16, %eax
2086; ALL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2087; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2088; ALL-NEXT:    vmovd %xmm0, %edx
2089; ALL-NEXT:    movzwl %dx, %edx
2090; ALL-NEXT:    orl %eax, %edx
2091; ALL-NEXT:    shlq $32, %rdx
2092; ALL-NEXT:    orq %rcx, %rdx
2093; ALL-NEXT:    vmovq %rdx, %xmm0
2094; ALL-NEXT:    retq
2095  %1 = fptrunc <4 x float> %a0 to <4 x half>
2096  %2 = bitcast <4 x half> %1 to <4 x i16>
2097  ret <4 x i16> %2
2098}
2099
2100define <8 x i16> @cvt_4f32_to_8i16_undef(<4 x float> %a0) nounwind {
2101; AVX1-LABEL: cvt_4f32_to_8i16_undef:
2102; AVX1:       # BB#0:
2103; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2104; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2105; AVX1-NEXT:    vmovd %xmm1, %eax
2106; AVX1-NEXT:    shll $16, %eax
2107; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2108; AVX1-NEXT:    vmovd %xmm1, %ecx
2109; AVX1-NEXT:    movzwl %cx, %ecx
2110; AVX1-NEXT:    orl %eax, %ecx
2111; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2112; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2113; AVX1-NEXT:    vmovd %xmm1, %eax
2114; AVX1-NEXT:    shll $16, %eax
2115; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2116; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2117; AVX1-NEXT:    vmovd %xmm0, %edx
2118; AVX1-NEXT:    movzwl %dx, %edx
2119; AVX1-NEXT:    orl %eax, %edx
2120; AVX1-NEXT:    shlq $32, %rdx
2121; AVX1-NEXT:    orq %rcx, %rdx
2122; AVX1-NEXT:    vmovq %rdx, %xmm0
2123; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2124; AVX1-NEXT:    retq
2125;
2126; AVX2-LABEL: cvt_4f32_to_8i16_undef:
2127; AVX2:       # BB#0:
2128; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2129; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2130; AVX2-NEXT:    vmovd %xmm1, %eax
2131; AVX2-NEXT:    shll $16, %eax
2132; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2133; AVX2-NEXT:    vmovd %xmm1, %ecx
2134; AVX2-NEXT:    movzwl %cx, %ecx
2135; AVX2-NEXT:    orl %eax, %ecx
2136; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2137; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2138; AVX2-NEXT:    vmovd %xmm1, %eax
2139; AVX2-NEXT:    shll $16, %eax
2140; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2141; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2142; AVX2-NEXT:    vmovd %xmm0, %edx
2143; AVX2-NEXT:    movzwl %dx, %edx
2144; AVX2-NEXT:    orl %eax, %edx
2145; AVX2-NEXT:    shlq $32, %rdx
2146; AVX2-NEXT:    orq %rcx, %rdx
2147; AVX2-NEXT:    vmovq %rdx, %xmm0
2148; AVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2149; AVX2-NEXT:    retq
2150;
2151; AVX512F-LABEL: cvt_4f32_to_8i16_undef:
2152; AVX512F:       # BB#0:
2153; AVX512F-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2154; AVX512F-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2155; AVX512F-NEXT:    vmovd %xmm1, %eax
2156; AVX512F-NEXT:    shll $16, %eax
2157; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2158; AVX512F-NEXT:    vmovd %xmm1, %ecx
2159; AVX512F-NEXT:    movzwl %cx, %ecx
2160; AVX512F-NEXT:    orl %eax, %ecx
2161; AVX512F-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2162; AVX512F-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2163; AVX512F-NEXT:    vmovd %xmm1, %eax
2164; AVX512F-NEXT:    shll $16, %eax
2165; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2166; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2167; AVX512F-NEXT:    vmovd %xmm0, %edx
2168; AVX512F-NEXT:    movzwl %dx, %edx
2169; AVX512F-NEXT:    orl %eax, %edx
2170; AVX512F-NEXT:    shlq $32, %rdx
2171; AVX512F-NEXT:    orq %rcx, %rdx
2172; AVX512F-NEXT:    vmovq %rdx, %xmm0
2173; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2174; AVX512F-NEXT:    retq
2175;
2176; AVX512VL-LABEL: cvt_4f32_to_8i16_undef:
2177; AVX512VL:       # BB#0:
2178; AVX512VL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2179; AVX512VL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2180; AVX512VL-NEXT:    vmovd %xmm1, %eax
2181; AVX512VL-NEXT:    shll $16, %eax
2182; AVX512VL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2183; AVX512VL-NEXT:    vmovd %xmm1, %ecx
2184; AVX512VL-NEXT:    movzwl %cx, %ecx
2185; AVX512VL-NEXT:    orl %eax, %ecx
2186; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2187; AVX512VL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2188; AVX512VL-NEXT:    vmovd %xmm1, %eax
2189; AVX512VL-NEXT:    shll $16, %eax
2190; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2191; AVX512VL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2192; AVX512VL-NEXT:    vmovd %xmm0, %edx
2193; AVX512VL-NEXT:    movzwl %dx, %edx
2194; AVX512VL-NEXT:    orl %eax, %edx
2195; AVX512VL-NEXT:    shlq $32, %rdx
2196; AVX512VL-NEXT:    orq %rcx, %rdx
2197; AVX512VL-NEXT:    vmovq %rdx, %xmm0
2198; AVX512VL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2199; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2200; AVX512VL-NEXT:    retq
2201  %1 = fptrunc <4 x float> %a0 to <4 x half>
2202  %2 = bitcast <4 x half> %1 to <4 x i16>
2203  %3 = shufflevector <4 x i16> %2, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
2204  ret <8 x i16> %3
2205}
2206
2207define <8 x i16> @cvt_4f32_to_8i16_zero(<4 x float> %a0) nounwind {
2208; AVX1-LABEL: cvt_4f32_to_8i16_zero:
2209; AVX1:       # BB#0:
2210; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2211; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2212; AVX1-NEXT:    vmovd %xmm1, %eax
2213; AVX1-NEXT:    shll $16, %eax
2214; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2215; AVX1-NEXT:    vmovd %xmm1, %ecx
2216; AVX1-NEXT:    movzwl %cx, %ecx
2217; AVX1-NEXT:    orl %eax, %ecx
2218; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2219; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2220; AVX1-NEXT:    vmovd %xmm1, %eax
2221; AVX1-NEXT:    shll $16, %eax
2222; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2223; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2224; AVX1-NEXT:    vmovd %xmm0, %edx
2225; AVX1-NEXT:    movzwl %dx, %edx
2226; AVX1-NEXT:    orl %eax, %edx
2227; AVX1-NEXT:    shlq $32, %rdx
2228; AVX1-NEXT:    orq %rcx, %rdx
2229; AVX1-NEXT:    vmovq %rdx, %xmm0
2230; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero
2231; AVX1-NEXT:    retq
2232;
2233; AVX2-LABEL: cvt_4f32_to_8i16_zero:
2234; AVX2:       # BB#0:
2235; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2236; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2237; AVX2-NEXT:    vmovd %xmm1, %eax
2238; AVX2-NEXT:    shll $16, %eax
2239; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2240; AVX2-NEXT:    vmovd %xmm1, %ecx
2241; AVX2-NEXT:    movzwl %cx, %ecx
2242; AVX2-NEXT:    orl %eax, %ecx
2243; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2244; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2245; AVX2-NEXT:    vmovd %xmm1, %eax
2246; AVX2-NEXT:    shll $16, %eax
2247; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2248; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2249; AVX2-NEXT:    vmovd %xmm0, %edx
2250; AVX2-NEXT:    movzwl %dx, %edx
2251; AVX2-NEXT:    orl %eax, %edx
2252; AVX2-NEXT:    shlq $32, %rdx
2253; AVX2-NEXT:    orq %rcx, %rdx
2254; AVX2-NEXT:    vmovq %rdx, %xmm0
2255; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero
2256; AVX2-NEXT:    retq
2257;
2258; AVX512F-LABEL: cvt_4f32_to_8i16_zero:
2259; AVX512F:       # BB#0:
2260; AVX512F-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2261; AVX512F-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2262; AVX512F-NEXT:    vmovd %xmm1, %eax
2263; AVX512F-NEXT:    shll $16, %eax
2264; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2265; AVX512F-NEXT:    vmovd %xmm1, %ecx
2266; AVX512F-NEXT:    movzwl %cx, %ecx
2267; AVX512F-NEXT:    orl %eax, %ecx
2268; AVX512F-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2269; AVX512F-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2270; AVX512F-NEXT:    vmovd %xmm1, %eax
2271; AVX512F-NEXT:    shll $16, %eax
2272; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2273; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2274; AVX512F-NEXT:    vmovd %xmm0, %edx
2275; AVX512F-NEXT:    movzwl %dx, %edx
2276; AVX512F-NEXT:    orl %eax, %edx
2277; AVX512F-NEXT:    shlq $32, %rdx
2278; AVX512F-NEXT:    orq %rcx, %rdx
2279; AVX512F-NEXT:    vmovq %rdx, %xmm0
2280; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero
2281; AVX512F-NEXT:    retq
2282;
2283; AVX512VL-LABEL: cvt_4f32_to_8i16_zero:
2284; AVX512VL:       # BB#0:
2285; AVX512VL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2286; AVX512VL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2287; AVX512VL-NEXT:    vmovd %xmm1, %eax
2288; AVX512VL-NEXT:    shll $16, %eax
2289; AVX512VL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2290; AVX512VL-NEXT:    vmovd %xmm1, %ecx
2291; AVX512VL-NEXT:    movzwl %cx, %ecx
2292; AVX512VL-NEXT:    orl %eax, %ecx
2293; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2294; AVX512VL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2295; AVX512VL-NEXT:    vmovd %xmm1, %eax
2296; AVX512VL-NEXT:    shll $16, %eax
2297; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2298; AVX512VL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2299; AVX512VL-NEXT:    vmovd %xmm0, %edx
2300; AVX512VL-NEXT:    movzwl %dx, %edx
2301; AVX512VL-NEXT:    orl %eax, %edx
2302; AVX512VL-NEXT:    shlq $32, %rdx
2303; AVX512VL-NEXT:    orq %rcx, %rdx
2304; AVX512VL-NEXT:    vmovq %rdx, %xmm0
2305; AVX512VL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2306; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]
2307; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
2308; AVX512VL-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
2309; AVX512VL-NEXT:    retq
2310  %1 = fptrunc <4 x float> %a0 to <4 x half>
2311  %2 = bitcast <4 x half> %1 to <4 x i16>
2312  %3 = shufflevector <4 x i16> %2, <4 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
2313  ret <8 x i16> %3
2314}
2315
2316define <8 x i16> @cvt_8f32_to_8i16(<8 x float> %a0) nounwind {
2317; ALL-LABEL: cvt_8f32_to_8i16:
2318; ALL:       # BB#0:
2319; ALL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2320; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2321; ALL-NEXT:    vmovd %xmm1, %eax
2322; ALL-NEXT:    shll $16, %eax
2323; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2324; ALL-NEXT:    vmovd %xmm1, %ecx
2325; ALL-NEXT:    movzwl %cx, %ecx
2326; ALL-NEXT:    orl %eax, %ecx
2327; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2328; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2329; ALL-NEXT:    vmovd %xmm1, %edx
2330; ALL-NEXT:    shll $16, %edx
2331; ALL-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
2332; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2333; ALL-NEXT:    vmovd %xmm1, %eax
2334; ALL-NEXT:    movzwl %ax, %eax
2335; ALL-NEXT:    orl %edx, %eax
2336; ALL-NEXT:    shlq $32, %rax
2337; ALL-NEXT:    orq %rcx, %rax
2338; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
2339; ALL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2340; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2341; ALL-NEXT:    vmovd %xmm1, %ecx
2342; ALL-NEXT:    shll $16, %ecx
2343; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2344; ALL-NEXT:    vmovd %xmm1, %edx
2345; ALL-NEXT:    movzwl %dx, %edx
2346; ALL-NEXT:    orl %ecx, %edx
2347; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2348; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2349; ALL-NEXT:    vmovd %xmm1, %ecx
2350; ALL-NEXT:    shll $16, %ecx
2351; ALL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2352; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2353; ALL-NEXT:    vmovd %xmm0, %esi
2354; ALL-NEXT:    movzwl %si, %esi
2355; ALL-NEXT:    orl %ecx, %esi
2356; ALL-NEXT:    shlq $32, %rsi
2357; ALL-NEXT:    orq %rdx, %rsi
2358; ALL-NEXT:    vmovq %rsi, %xmm0
2359; ALL-NEXT:    vmovq %rax, %xmm1
2360; ALL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
2361; ALL-NEXT:    vzeroupper
2362; ALL-NEXT:    retq
2363  %1 = fptrunc <8 x float> %a0 to <8 x half>
2364  %2 = bitcast <8 x half> %1 to <8 x i16>
2365  ret <8 x i16> %2
2366}
2367
2368define <16 x i16> @cvt_16f32_to_16i16(<16 x float> %a0) nounwind {
2369; AVX1-LABEL: cvt_16f32_to_16i16:
2370; AVX1:       # BB#0:
2371; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm2
2372; AVX1-NEXT:    vmovd %xmm2, %eax
2373; AVX1-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
2374; AVX1-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2375; AVX1-NEXT:    vmovd %eax, %xmm3
2376; AVX1-NEXT:    vmovd %xmm2, %eax
2377; AVX1-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
2378; AVX1-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2379; AVX1-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
2380; AVX1-NEXT:    vmovd %xmm2, %eax
2381; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
2382; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
2383; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2384; AVX1-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
2385; AVX1-NEXT:    vmovd %xmm1, %eax
2386; AVX1-NEXT:    vcvtps2ph $4, %xmm2, %xmm1
2387; AVX1-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
2388; AVX1-NEXT:    vmovd %xmm1, %eax
2389; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
2390; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2391; AVX1-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
2392; AVX1-NEXT:    vmovd %xmm1, %eax
2393; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm2[1,0]
2394; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2395; AVX1-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
2396; AVX1-NEXT:    vmovd %xmm1, %eax
2397; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2398; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
2399; AVX1-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2400; AVX1-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm3
2401; AVX1-NEXT:    vmovd %xmm2, %eax
2402; AVX1-NEXT:    vpinsrw $7, %eax, %xmm3, %xmm2
2403; AVX1-NEXT:    vmovd %xmm1, %eax
2404; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2405; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2406; AVX1-NEXT:    vmovd %eax, %xmm3
2407; AVX1-NEXT:    vmovd %xmm1, %eax
2408; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
2409; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2410; AVX1-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
2411; AVX1-NEXT:    vmovd %xmm1, %eax
2412; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
2413; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
2414; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2415; AVX1-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
2416; AVX1-NEXT:    vmovd %xmm0, %eax
2417; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm0
2418; AVX1-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
2419; AVX1-NEXT:    vmovd %xmm0, %eax
2420; AVX1-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
2421; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2422; AVX1-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
2423; AVX1-NEXT:    vmovd %xmm0, %eax
2424; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm1[3,1,2,3]
2425; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2426; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
2427; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2428; AVX1-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
2429; AVX1-NEXT:    vmovd %xmm1, %eax
2430; AVX1-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm1
2431; AVX1-NEXT:    vmovd %xmm0, %eax
2432; AVX1-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
2433; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
2434; AVX1-NEXT:    retq
2435;
2436; AVX2-LABEL: cvt_16f32_to_16i16:
2437; AVX2:       # BB#0:
2438; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm2
2439; AVX2-NEXT:    vmovd %xmm2, %eax
2440; AVX2-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
2441; AVX2-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2442; AVX2-NEXT:    vmovd %eax, %xmm3
2443; AVX2-NEXT:    vmovd %xmm2, %eax
2444; AVX2-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
2445; AVX2-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2446; AVX2-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
2447; AVX2-NEXT:    vmovd %xmm2, %eax
2448; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
2449; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
2450; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2451; AVX2-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
2452; AVX2-NEXT:    vmovd %xmm1, %eax
2453; AVX2-NEXT:    vcvtps2ph $4, %xmm2, %xmm1
2454; AVX2-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
2455; AVX2-NEXT:    vmovd %xmm1, %eax
2456; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
2457; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2458; AVX2-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
2459; AVX2-NEXT:    vmovd %xmm1, %eax
2460; AVX2-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm2[1,0]
2461; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2462; AVX2-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
2463; AVX2-NEXT:    vmovd %xmm1, %eax
2464; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2465; AVX2-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
2466; AVX2-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2467; AVX2-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm3
2468; AVX2-NEXT:    vmovd %xmm2, %eax
2469; AVX2-NEXT:    vpinsrw $7, %eax, %xmm3, %xmm2
2470; AVX2-NEXT:    vmovd %xmm1, %eax
2471; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2472; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2473; AVX2-NEXT:    vmovd %eax, %xmm3
2474; AVX2-NEXT:    vmovd %xmm1, %eax
2475; AVX2-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
2476; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2477; AVX2-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
2478; AVX2-NEXT:    vmovd %xmm1, %eax
2479; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
2480; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
2481; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2482; AVX2-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
2483; AVX2-NEXT:    vmovd %xmm0, %eax
2484; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm0
2485; AVX2-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
2486; AVX2-NEXT:    vmovd %xmm0, %eax
2487; AVX2-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
2488; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2489; AVX2-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
2490; AVX2-NEXT:    vmovd %xmm0, %eax
2491; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm1[3,1,2,3]
2492; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2493; AVX2-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
2494; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2495; AVX2-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
2496; AVX2-NEXT:    vmovd %xmm1, %eax
2497; AVX2-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm1
2498; AVX2-NEXT:    vmovd %xmm0, %eax
2499; AVX2-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
2500; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
2501; AVX2-NEXT:    retq
2502;
2503; AVX512-LABEL: cvt_16f32_to_16i16:
2504; AVX512:       # BB#0:
2505; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
2506; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm2
2507; AVX512-NEXT:    vmovd %xmm2, %eax
2508; AVX512-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
2509; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2510; AVX512-NEXT:    vmovd %eax, %xmm3
2511; AVX512-NEXT:    vmovd %xmm2, %eax
2512; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
2513; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2514; AVX512-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
2515; AVX512-NEXT:    vmovd %xmm2, %eax
2516; AVX512-NEXT:    vextractf128 $1, %ymm1, %xmm2
2517; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
2518; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2519; AVX512-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
2520; AVX512-NEXT:    vmovd %xmm1, %eax
2521; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm1
2522; AVX512-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
2523; AVX512-NEXT:    vmovd %xmm1, %eax
2524; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
2525; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2526; AVX512-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
2527; AVX512-NEXT:    vmovd %xmm1, %eax
2528; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm2[1,0]
2529; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2530; AVX512-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
2531; AVX512-NEXT:    vmovd %xmm1, %eax
2532; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2533; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
2534; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2535; AVX512-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm3
2536; AVX512-NEXT:    vmovd %xmm2, %eax
2537; AVX512-NEXT:    vpinsrw $7, %eax, %xmm3, %xmm2
2538; AVX512-NEXT:    vmovd %xmm1, %eax
2539; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2540; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2541; AVX512-NEXT:    vmovd %eax, %xmm3
2542; AVX512-NEXT:    vmovd %xmm1, %eax
2543; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
2544; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2545; AVX512-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
2546; AVX512-NEXT:    vmovd %xmm1, %eax
2547; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
2548; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
2549; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2550; AVX512-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
2551; AVX512-NEXT:    vmovd %xmm0, %eax
2552; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm0
2553; AVX512-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
2554; AVX512-NEXT:    vmovd %xmm0, %eax
2555; AVX512-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
2556; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2557; AVX512-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
2558; AVX512-NEXT:    vmovd %xmm0, %eax
2559; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm1[1,0]
2560; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2561; AVX512-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
2562; AVX512-NEXT:    vmovd %xmm0, %eax
2563; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm1[3,1,2,3]
2564; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2565; AVX512-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm1
2566; AVX512-NEXT:    vmovd %xmm0, %eax
2567; AVX512-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
2568; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
2569; AVX512-NEXT:    retq
2570  %1 = fptrunc <16 x float> %a0 to <16 x half>
2571  %2 = bitcast <16 x half> %1 to <16 x i16>
2572  ret <16 x i16> %2
2573}
2574
2575;
2576; Float to Half (Store)
2577;
2578
2579define void @store_cvt_f32_to_i16(float %a0, i16* %a1) nounwind {
2580; ALL-LABEL: store_cvt_f32_to_i16:
2581; ALL:       # BB#0:
2582; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2583; ALL-NEXT:    vmovd %xmm0, %eax
2584; ALL-NEXT:    movw %ax, (%rdi)
2585; ALL-NEXT:    retq
2586  %1 = fptrunc float %a0 to half
2587  %2 = bitcast half %1 to i16
2588  store i16 %2, i16* %a1
2589  ret void
2590}
2591
2592define void @store_cvt_4f32_to_4i16(<4 x float> %a0, <4 x i16>* %a1) nounwind {
2593; ALL-LABEL: store_cvt_4f32_to_4i16:
2594; ALL:       # BB#0:
2595; ALL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2596; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2597; ALL-NEXT:    vmovd %xmm1, %eax
2598; ALL-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
2599; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2600; ALL-NEXT:    vmovd %xmm1, %ecx
2601; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2602; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2603; ALL-NEXT:    vmovd %xmm1, %edx
2604; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2605; ALL-NEXT:    vmovd %xmm0, %esi
2606; ALL-NEXT:    movw %si, (%rdi)
2607; ALL-NEXT:    movw %dx, 6(%rdi)
2608; ALL-NEXT:    movw %cx, 4(%rdi)
2609; ALL-NEXT:    movw %ax, 2(%rdi)
2610; ALL-NEXT:    retq
2611  %1 = fptrunc <4 x float> %a0 to <4 x half>
2612  %2 = bitcast <4 x half> %1 to <4 x i16>
2613  store <4 x i16> %2, <4 x i16>* %a1
2614  ret void
2615}
2616
2617define void @store_cvt_4f32_to_8i16_undef(<4 x float> %a0, <8 x i16>* %a1) nounwind {
2618; AVX1-LABEL: store_cvt_4f32_to_8i16_undef:
2619; AVX1:       # BB#0:
2620; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2621; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2622; AVX1-NEXT:    vmovd %xmm1, %eax
2623; AVX1-NEXT:    shll $16, %eax
2624; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2625; AVX1-NEXT:    vmovd %xmm1, %ecx
2626; AVX1-NEXT:    movzwl %cx, %ecx
2627; AVX1-NEXT:    orl %eax, %ecx
2628; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2629; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2630; AVX1-NEXT:    vmovd %xmm1, %eax
2631; AVX1-NEXT:    shll $16, %eax
2632; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2633; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2634; AVX1-NEXT:    vmovd %xmm0, %edx
2635; AVX1-NEXT:    movzwl %dx, %edx
2636; AVX1-NEXT:    orl %eax, %edx
2637; AVX1-NEXT:    shlq $32, %rdx
2638; AVX1-NEXT:    orq %rcx, %rdx
2639; AVX1-NEXT:    vmovq %rdx, %xmm0
2640; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2641; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
2642; AVX1-NEXT:    retq
2643;
2644; AVX2-LABEL: store_cvt_4f32_to_8i16_undef:
2645; AVX2:       # BB#0:
2646; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2647; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2648; AVX2-NEXT:    vmovd %xmm1, %eax
2649; AVX2-NEXT:    shll $16, %eax
2650; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2651; AVX2-NEXT:    vmovd %xmm1, %ecx
2652; AVX2-NEXT:    movzwl %cx, %ecx
2653; AVX2-NEXT:    orl %eax, %ecx
2654; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2655; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2656; AVX2-NEXT:    vmovd %xmm1, %eax
2657; AVX2-NEXT:    shll $16, %eax
2658; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2659; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2660; AVX2-NEXT:    vmovd %xmm0, %edx
2661; AVX2-NEXT:    movzwl %dx, %edx
2662; AVX2-NEXT:    orl %eax, %edx
2663; AVX2-NEXT:    shlq $32, %rdx
2664; AVX2-NEXT:    orq %rcx, %rdx
2665; AVX2-NEXT:    vmovq %rdx, %xmm0
2666; AVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2667; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
2668; AVX2-NEXT:    retq
2669;
2670; AVX512F-LABEL: store_cvt_4f32_to_8i16_undef:
2671; AVX512F:       # BB#0:
2672; AVX512F-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2673; AVX512F-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2674; AVX512F-NEXT:    vmovd %xmm1, %eax
2675; AVX512F-NEXT:    shll $16, %eax
2676; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2677; AVX512F-NEXT:    vmovd %xmm1, %ecx
2678; AVX512F-NEXT:    movzwl %cx, %ecx
2679; AVX512F-NEXT:    orl %eax, %ecx
2680; AVX512F-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2681; AVX512F-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2682; AVX512F-NEXT:    vmovd %xmm1, %eax
2683; AVX512F-NEXT:    shll $16, %eax
2684; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2685; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2686; AVX512F-NEXT:    vmovd %xmm0, %edx
2687; AVX512F-NEXT:    movzwl %dx, %edx
2688; AVX512F-NEXT:    orl %eax, %edx
2689; AVX512F-NEXT:    shlq $32, %rdx
2690; AVX512F-NEXT:    orq %rcx, %rdx
2691; AVX512F-NEXT:    vmovq %rdx, %xmm0
2692; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2693; AVX512F-NEXT:    vmovdqa %xmm0, (%rdi)
2694; AVX512F-NEXT:    retq
2695;
2696; AVX512VL-LABEL: store_cvt_4f32_to_8i16_undef:
2697; AVX512VL:       # BB#0:
2698; AVX512VL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2699; AVX512VL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2700; AVX512VL-NEXT:    vmovd %xmm1, %eax
2701; AVX512VL-NEXT:    shll $16, %eax
2702; AVX512VL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2703; AVX512VL-NEXT:    vmovd %xmm1, %ecx
2704; AVX512VL-NEXT:    movzwl %cx, %ecx
2705; AVX512VL-NEXT:    orl %eax, %ecx
2706; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2707; AVX512VL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2708; AVX512VL-NEXT:    vmovd %xmm1, %eax
2709; AVX512VL-NEXT:    shll $16, %eax
2710; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2711; AVX512VL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2712; AVX512VL-NEXT:    vmovd %xmm0, %edx
2713; AVX512VL-NEXT:    movzwl %dx, %edx
2714; AVX512VL-NEXT:    orl %eax, %edx
2715; AVX512VL-NEXT:    shlq $32, %rdx
2716; AVX512VL-NEXT:    orq %rcx, %rdx
2717; AVX512VL-NEXT:    vmovq %rdx, %xmm0
2718; AVX512VL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2719; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2720; AVX512VL-NEXT:    vmovdqa %xmm0, (%rdi)
2721; AVX512VL-NEXT:    retq
2722  %1 = fptrunc <4 x float> %a0 to <4 x half>
2723  %2 = bitcast <4 x half> %1 to <4 x i16>
2724  %3 = shufflevector <4 x i16> %2, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
2725  store <8 x i16> %3, <8 x i16>* %a1
2726  ret void
2727}
2728
2729define void @store_cvt_4f32_to_8i16_zero(<4 x float> %a0, <8 x i16>* %a1) nounwind {
2730; AVX1-LABEL: store_cvt_4f32_to_8i16_zero:
2731; AVX1:       # BB#0:
2732; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2733; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2734; AVX1-NEXT:    vmovd %xmm1, %eax
2735; AVX1-NEXT:    shll $16, %eax
2736; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2737; AVX1-NEXT:    vmovd %xmm1, %ecx
2738; AVX1-NEXT:    movzwl %cx, %ecx
2739; AVX1-NEXT:    orl %eax, %ecx
2740; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2741; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2742; AVX1-NEXT:    vmovd %xmm1, %eax
2743; AVX1-NEXT:    shll $16, %eax
2744; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2745; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2746; AVX1-NEXT:    vmovd %xmm0, %edx
2747; AVX1-NEXT:    movzwl %dx, %edx
2748; AVX1-NEXT:    orl %eax, %edx
2749; AVX1-NEXT:    shlq $32, %rdx
2750; AVX1-NEXT:    orq %rcx, %rdx
2751; AVX1-NEXT:    vmovq %rdx, %xmm0
2752; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero
2753; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
2754; AVX1-NEXT:    retq
2755;
2756; AVX2-LABEL: store_cvt_4f32_to_8i16_zero:
2757; AVX2:       # BB#0:
2758; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2759; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2760; AVX2-NEXT:    vmovd %xmm1, %eax
2761; AVX2-NEXT:    shll $16, %eax
2762; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2763; AVX2-NEXT:    vmovd %xmm1, %ecx
2764; AVX2-NEXT:    movzwl %cx, %ecx
2765; AVX2-NEXT:    orl %eax, %ecx
2766; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2767; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2768; AVX2-NEXT:    vmovd %xmm1, %eax
2769; AVX2-NEXT:    shll $16, %eax
2770; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2771; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2772; AVX2-NEXT:    vmovd %xmm0, %edx
2773; AVX2-NEXT:    movzwl %dx, %edx
2774; AVX2-NEXT:    orl %eax, %edx
2775; AVX2-NEXT:    shlq $32, %rdx
2776; AVX2-NEXT:    orq %rcx, %rdx
2777; AVX2-NEXT:    vmovq %rdx, %xmm0
2778; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero
2779; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
2780; AVX2-NEXT:    retq
2781;
2782; AVX512F-LABEL: store_cvt_4f32_to_8i16_zero:
2783; AVX512F:       # BB#0:
2784; AVX512F-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2785; AVX512F-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2786; AVX512F-NEXT:    vmovd %xmm1, %eax
2787; AVX512F-NEXT:    shll $16, %eax
2788; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2789; AVX512F-NEXT:    vmovd %xmm1, %ecx
2790; AVX512F-NEXT:    movzwl %cx, %ecx
2791; AVX512F-NEXT:    orl %eax, %ecx
2792; AVX512F-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2793; AVX512F-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2794; AVX512F-NEXT:    vmovd %xmm1, %eax
2795; AVX512F-NEXT:    shll $16, %eax
2796; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2797; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2798; AVX512F-NEXT:    vmovd %xmm0, %edx
2799; AVX512F-NEXT:    movzwl %dx, %edx
2800; AVX512F-NEXT:    orl %eax, %edx
2801; AVX512F-NEXT:    shlq $32, %rdx
2802; AVX512F-NEXT:    orq %rcx, %rdx
2803; AVX512F-NEXT:    vmovq %rdx, %xmm0
2804; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero
2805; AVX512F-NEXT:    vmovdqa %xmm0, (%rdi)
2806; AVX512F-NEXT:    retq
2807;
2808; AVX512VL-LABEL: store_cvt_4f32_to_8i16_zero:
2809; AVX512VL:       # BB#0:
2810; AVX512VL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2811; AVX512VL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2812; AVX512VL-NEXT:    vmovd %xmm1, %eax
2813; AVX512VL-NEXT:    shll $16, %eax
2814; AVX512VL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2815; AVX512VL-NEXT:    vmovd %xmm1, %ecx
2816; AVX512VL-NEXT:    movzwl %cx, %ecx
2817; AVX512VL-NEXT:    orl %eax, %ecx
2818; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2819; AVX512VL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2820; AVX512VL-NEXT:    vmovd %xmm1, %eax
2821; AVX512VL-NEXT:    shll $16, %eax
2822; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2823; AVX512VL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2824; AVX512VL-NEXT:    vmovd %xmm0, %edx
2825; AVX512VL-NEXT:    movzwl %dx, %edx
2826; AVX512VL-NEXT:    orl %eax, %edx
2827; AVX512VL-NEXT:    shlq $32, %rdx
2828; AVX512VL-NEXT:    orq %rcx, %rdx
2829; AVX512VL-NEXT:    vmovq %rdx, %xmm0
2830; AVX512VL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2831; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]
2832; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
2833; AVX512VL-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
2834; AVX512VL-NEXT:    vmovdqa %xmm0, (%rdi)
2835; AVX512VL-NEXT:    retq
2836  %1 = fptrunc <4 x float> %a0 to <4 x half>
2837  %2 = bitcast <4 x half> %1 to <4 x i16>
2838  %3 = shufflevector <4 x i16> %2, <4 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
2839  store <8 x i16> %3, <8 x i16>* %a1
2840  ret void
2841}
2842
2843define void @store_cvt_8f32_to_8i16(<8 x float> %a0, <8 x i16>* %a1) nounwind {
2844; ALL-LABEL: store_cvt_8f32_to_8i16:
2845; ALL:       # BB#0:
2846; ALL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2847; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2848; ALL-NEXT:    vmovd %xmm1, %r8d
2849; ALL-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
2850; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2851; ALL-NEXT:    vmovd %xmm1, %r9d
2852; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2853; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2854; ALL-NEXT:    vmovd %xmm1, %r10d
2855; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm1
2856; ALL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
2857; ALL-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2858; ALL-NEXT:    vmovd %xmm2, %r11d
2859; ALL-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
2860; ALL-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2861; ALL-NEXT:    vmovd %xmm2, %eax
2862; ALL-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
2863; ALL-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2864; ALL-NEXT:    vmovd %xmm2, %ecx
2865; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2866; ALL-NEXT:    vmovd %xmm0, %edx
2867; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm0
2868; ALL-NEXT:    vmovd %xmm0, %esi
2869; ALL-NEXT:    movw %si, 8(%rdi)
2870; ALL-NEXT:    movw %dx, (%rdi)
2871; ALL-NEXT:    movw %cx, 14(%rdi)
2872; ALL-NEXT:    movw %ax, 12(%rdi)
2873; ALL-NEXT:    movw %r11w, 10(%rdi)
2874; ALL-NEXT:    movw %r10w, 6(%rdi)
2875; ALL-NEXT:    movw %r9w, 4(%rdi)
2876; ALL-NEXT:    movw %r8w, 2(%rdi)
2877; ALL-NEXT:    vzeroupper
2878; ALL-NEXT:    retq
2879  %1 = fptrunc <8 x float> %a0 to <8 x half>
2880  %2 = bitcast <8 x half> %1 to <8 x i16>
2881  store <8 x i16> %2, <8 x i16>* %a1
2882  ret void
2883}
2884
2885define void @store_cvt_16f32_to_16i16(<16 x float> %a0, <16 x i16>* %a1) nounwind {
2886; AVX1-LABEL: store_cvt_16f32_to_16i16:
2887; AVX1:       # BB#0:
2888; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
2889; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
2890; AVX1-NEXT:    vcvtps2ph $4, %xmm3, %xmm4
2891; AVX1-NEXT:    vmovd %xmm4, %eax
2892; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm4
2893; AVX1-NEXT:    movw %ax, 24(%rdi)
2894; AVX1-NEXT:    vmovd %xmm4, %eax
2895; AVX1-NEXT:    vcvtps2ph $4, %xmm2, %xmm4
2896; AVX1-NEXT:    movw %ax, 16(%rdi)
2897; AVX1-NEXT:    vmovd %xmm4, %eax
2898; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm4
2899; AVX1-NEXT:    movw %ax, 8(%rdi)
2900; AVX1-NEXT:    vmovd %xmm4, %eax
2901; AVX1-NEXT:    vpermilps {{.*#+}} xmm4 = xmm3[3,1,2,3]
2902; AVX1-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2903; AVX1-NEXT:    movw %ax, (%rdi)
2904; AVX1-NEXT:    vmovd %xmm4, %eax
2905; AVX1-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm3[1,0]
2906; AVX1-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2907; AVX1-NEXT:    movw %ax, 30(%rdi)
2908; AVX1-NEXT:    vmovd %xmm4, %eax
2909; AVX1-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
2910; AVX1-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2911; AVX1-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm3[1,1,3,3]
2912; AVX1-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2913; AVX1-NEXT:    movw %ax, 28(%rdi)
2914; AVX1-NEXT:    vmovd %xmm3, %eax
2915; AVX1-NEXT:    vpermilps {{.*#+}} xmm3 = xmm1[3,1,2,3]
2916; AVX1-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2917; AVX1-NEXT:    movw %ax, 26(%rdi)
2918; AVX1-NEXT:    vmovd %xmm3, %eax
2919; AVX1-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
2920; AVX1-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2921; AVX1-NEXT:    movw %ax, 22(%rdi)
2922; AVX1-NEXT:    vmovd %xmm3, %eax
2923; AVX1-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
2924; AVX1-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2925; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
2926; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2927; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm1[1,1,3,3]
2928; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2929; AVX1-NEXT:    movw %ax, 20(%rdi)
2930; AVX1-NEXT:    vmovd %xmm1, %eax
2931; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm2[3,1,2,3]
2932; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2933; AVX1-NEXT:    movw %ax, 18(%rdi)
2934; AVX1-NEXT:    vmovd %xmm1, %eax
2935; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
2936; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2937; AVX1-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm2[1,0]
2938; AVX1-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2939; AVX1-NEXT:    movw %ax, 14(%rdi)
2940; AVX1-NEXT:    vmovd %xmm2, %eax
2941; AVX1-NEXT:    movw %ax, 12(%rdi)
2942; AVX1-NEXT:    vmovd %xmm1, %eax
2943; AVX1-NEXT:    movw %ax, 10(%rdi)
2944; AVX1-NEXT:    vmovd %xmm0, %eax
2945; AVX1-NEXT:    movw %ax, 6(%rdi)
2946; AVX1-NEXT:    vmovd %xmm3, %eax
2947; AVX1-NEXT:    movw %ax, 4(%rdi)
2948; AVX1-NEXT:    vmovd %xmm4, %eax
2949; AVX1-NEXT:    movw %ax, 2(%rdi)
2950; AVX1-NEXT:    vzeroupper
2951; AVX1-NEXT:    retq
2952;
2953; AVX2-LABEL: store_cvt_16f32_to_16i16:
2954; AVX2:       # BB#0:
2955; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
2956; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm3
2957; AVX2-NEXT:    vcvtps2ph $4, %xmm3, %xmm4
2958; AVX2-NEXT:    vmovd %xmm4, %eax
2959; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm4
2960; AVX2-NEXT:    movw %ax, 24(%rdi)
2961; AVX2-NEXT:    vmovd %xmm4, %eax
2962; AVX2-NEXT:    vcvtps2ph $4, %xmm2, %xmm4
2963; AVX2-NEXT:    movw %ax, 16(%rdi)
2964; AVX2-NEXT:    vmovd %xmm4, %eax
2965; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm4
2966; AVX2-NEXT:    movw %ax, 8(%rdi)
2967; AVX2-NEXT:    vmovd %xmm4, %eax
2968; AVX2-NEXT:    vpermilps {{.*#+}} xmm4 = xmm3[3,1,2,3]
2969; AVX2-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2970; AVX2-NEXT:    movw %ax, (%rdi)
2971; AVX2-NEXT:    vmovd %xmm4, %eax
2972; AVX2-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm3[1,0]
2973; AVX2-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2974; AVX2-NEXT:    movw %ax, 30(%rdi)
2975; AVX2-NEXT:    vmovd %xmm4, %eax
2976; AVX2-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
2977; AVX2-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2978; AVX2-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm3[1,1,3,3]
2979; AVX2-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2980; AVX2-NEXT:    movw %ax, 28(%rdi)
2981; AVX2-NEXT:    vmovd %xmm3, %eax
2982; AVX2-NEXT:    vpermilps {{.*#+}} xmm3 = xmm1[3,1,2,3]
2983; AVX2-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2984; AVX2-NEXT:    movw %ax, 26(%rdi)
2985; AVX2-NEXT:    vmovd %xmm3, %eax
2986; AVX2-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
2987; AVX2-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2988; AVX2-NEXT:    movw %ax, 22(%rdi)
2989; AVX2-NEXT:    vmovd %xmm3, %eax
2990; AVX2-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
2991; AVX2-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2992; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
2993; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2994; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm1[1,1,3,3]
2995; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2996; AVX2-NEXT:    movw %ax, 20(%rdi)
2997; AVX2-NEXT:    vmovd %xmm1, %eax
2998; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm2[3,1,2,3]
2999; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
3000; AVX2-NEXT:    movw %ax, 18(%rdi)
3001; AVX2-NEXT:    vmovd %xmm1, %eax
3002; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
3003; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
3004; AVX2-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm2[1,0]
3005; AVX2-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
3006; AVX2-NEXT:    movw %ax, 14(%rdi)
3007; AVX2-NEXT:    vmovd %xmm2, %eax
3008; AVX2-NEXT:    movw %ax, 12(%rdi)
3009; AVX2-NEXT:    vmovd %xmm1, %eax
3010; AVX2-NEXT:    movw %ax, 10(%rdi)
3011; AVX2-NEXT:    vmovd %xmm0, %eax
3012; AVX2-NEXT:    movw %ax, 6(%rdi)
3013; AVX2-NEXT:    vmovd %xmm3, %eax
3014; AVX2-NEXT:    movw %ax, 4(%rdi)
3015; AVX2-NEXT:    vmovd %xmm4, %eax
3016; AVX2-NEXT:    movw %ax, 2(%rdi)
3017; AVX2-NEXT:    vzeroupper
3018; AVX2-NEXT:    retq
3019;
3020; AVX512-LABEL: store_cvt_16f32_to_16i16:
3021; AVX512:       # BB#0:
3022; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
3023; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
3024; AVX512-NEXT:    vextractf128 $1, %ymm2, %xmm3
3025; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm4
3026; AVX512-NEXT:    vmovd %xmm4, %eax
3027; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm4
3028; AVX512-NEXT:    movw %ax, 24(%rdi)
3029; AVX512-NEXT:    vmovd %xmm4, %eax
3030; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm4
3031; AVX512-NEXT:    movw %ax, 16(%rdi)
3032; AVX512-NEXT:    vmovd %xmm4, %eax
3033; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm4
3034; AVX512-NEXT:    movw %ax, 8(%rdi)
3035; AVX512-NEXT:    vmovd %xmm4, %eax
3036; AVX512-NEXT:    vpermilps {{.*#+}} xmm4 = xmm3[3,1,2,3]
3037; AVX512-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
3038; AVX512-NEXT:    movw %ax, (%rdi)
3039; AVX512-NEXT:    vmovd %xmm4, %eax
3040; AVX512-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm3[1,0]
3041; AVX512-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
3042; AVX512-NEXT:    movw %ax, 30(%rdi)
3043; AVX512-NEXT:    vmovd %xmm4, %eax
3044; AVX512-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
3045; AVX512-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
3046; AVX512-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm3[1,1,3,3]
3047; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
3048; AVX512-NEXT:    movw %ax, 28(%rdi)
3049; AVX512-NEXT:    vmovd %xmm3, %eax
3050; AVX512-NEXT:    vpermilps {{.*#+}} xmm3 = xmm2[3,1,2,3]
3051; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
3052; AVX512-NEXT:    movw %ax, 26(%rdi)
3053; AVX512-NEXT:    vmovd %xmm3, %eax
3054; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
3055; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
3056; AVX512-NEXT:    movw %ax, 22(%rdi)
3057; AVX512-NEXT:    vmovd %xmm3, %eax
3058; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
3059; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
3060; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
3061; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
3062; AVX512-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm2[1,1,3,3]
3063; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
3064; AVX512-NEXT:    movw %ax, 20(%rdi)
3065; AVX512-NEXT:    vmovd %xmm2, %eax
3066; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
3067; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
3068; AVX512-NEXT:    movw %ax, 18(%rdi)
3069; AVX512-NEXT:    vmovd %xmm2, %eax
3070; AVX512-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
3071; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
3072; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
3073; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
3074; AVX512-NEXT:    movw %ax, 14(%rdi)
3075; AVX512-NEXT:    vmovd %xmm1, %eax
3076; AVX512-NEXT:    movw %ax, 12(%rdi)
3077; AVX512-NEXT:    vmovd %xmm2, %eax
3078; AVX512-NEXT:    movw %ax, 10(%rdi)
3079; AVX512-NEXT:    vmovd %xmm0, %eax
3080; AVX512-NEXT:    movw %ax, 6(%rdi)
3081; AVX512-NEXT:    vmovd %xmm3, %eax
3082; AVX512-NEXT:    movw %ax, 4(%rdi)
3083; AVX512-NEXT:    vmovd %xmm4, %eax
3084; AVX512-NEXT:    movw %ax, 2(%rdi)
3085; AVX512-NEXT:    vzeroupper
3086; AVX512-NEXT:    retq
3087  %1 = fptrunc <16 x float> %a0 to <16 x half>
3088  %2 = bitcast <16 x half> %1 to <16 x i16>
3089  store <16 x i16> %2, <16 x i16>* %a1
3090  ret void
3091}
3092
3093;
3094; Double to Half
3095;
3096
3097define i16 @cvt_f64_to_i16(double %a0) nounwind {
3098; ALL-LABEL: cvt_f64_to_i16:
3099; ALL:       # BB#0:
3100; ALL-NEXT:    jmp __truncdfhf2 # TAILCALL
3101  %1 = fptrunc double %a0 to half
3102  %2 = bitcast half %1 to i16
3103  ret i16 %2
3104}
3105
3106define <2 x i16> @cvt_2f64_to_2i16(<2 x double> %a0) nounwind {
3107; ALL-LABEL: cvt_2f64_to_2i16:
3108; ALL:       # BB#0:
3109; ALL-NEXT:    pushq %rbx
3110; ALL-NEXT:    subq $16, %rsp
3111; ALL-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3112; ALL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3113; ALL-NEXT:    callq __truncdfhf2
3114; ALL-NEXT:    movl %eax, %ebx
3115; ALL-NEXT:    shll $16, %ebx
3116; ALL-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3117; ALL-NEXT:    callq __truncdfhf2
3118; ALL-NEXT:    movzwl %ax, %eax
3119; ALL-NEXT:    orl %ebx, %eax
3120; ALL-NEXT:    vmovd %eax, %xmm0
3121; ALL-NEXT:    addq $16, %rsp
3122; ALL-NEXT:    popq %rbx
3123; ALL-NEXT:    retq
3124  %1 = fptrunc <2 x double> %a0 to <2 x half>
3125  %2 = bitcast <2 x half> %1 to <2 x i16>
3126  ret <2 x i16> %2
3127}
3128
3129define <4 x i16> @cvt_4f64_to_4i16(<4 x double> %a0) nounwind {
3130; AVX1-LABEL: cvt_4f64_to_4i16:
3131; AVX1:       # BB#0:
3132; AVX1-NEXT:    pushq %r14
3133; AVX1-NEXT:    pushq %rbx
3134; AVX1-NEXT:    subq $40, %rsp
3135; AVX1-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3136; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3137; AVX1-NEXT:    vzeroupper
3138; AVX1-NEXT:    callq __truncdfhf2
3139; AVX1-NEXT:    movl %eax, %ebx
3140; AVX1-NEXT:    shll $16, %ebx
3141; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3142; AVX1-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3143; AVX1-NEXT:    vzeroupper
3144; AVX1-NEXT:    callq __truncdfhf2
3145; AVX1-NEXT:    movzwl %ax, %r14d
3146; AVX1-NEXT:    orl %ebx, %r14d
3147; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3148; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3149; AVX1-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3150; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3151; AVX1-NEXT:    vzeroupper
3152; AVX1-NEXT:    callq __truncdfhf2
3153; AVX1-NEXT:    movl %eax, %ebx
3154; AVX1-NEXT:    shll $16, %ebx
3155; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3156; AVX1-NEXT:    callq __truncdfhf2
3157; AVX1-NEXT:    movzwl %ax, %eax
3158; AVX1-NEXT:    orl %ebx, %eax
3159; AVX1-NEXT:    shlq $32, %rax
3160; AVX1-NEXT:    orq %r14, %rax
3161; AVX1-NEXT:    vmovq %rax, %xmm0
3162; AVX1-NEXT:    addq $40, %rsp
3163; AVX1-NEXT:    popq %rbx
3164; AVX1-NEXT:    popq %r14
3165; AVX1-NEXT:    retq
3166;
3167; AVX2-LABEL: cvt_4f64_to_4i16:
3168; AVX2:       # BB#0:
3169; AVX2-NEXT:    pushq %r14
3170; AVX2-NEXT:    pushq %rbx
3171; AVX2-NEXT:    subq $40, %rsp
3172; AVX2-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3173; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3174; AVX2-NEXT:    vzeroupper
3175; AVX2-NEXT:    callq __truncdfhf2
3176; AVX2-NEXT:    movl %eax, %ebx
3177; AVX2-NEXT:    shll $16, %ebx
3178; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3179; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3180; AVX2-NEXT:    vzeroupper
3181; AVX2-NEXT:    callq __truncdfhf2
3182; AVX2-NEXT:    movzwl %ax, %r14d
3183; AVX2-NEXT:    orl %ebx, %r14d
3184; AVX2-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3185; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
3186; AVX2-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3187; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3188; AVX2-NEXT:    vzeroupper
3189; AVX2-NEXT:    callq __truncdfhf2
3190; AVX2-NEXT:    movl %eax, %ebx
3191; AVX2-NEXT:    shll $16, %ebx
3192; AVX2-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3193; AVX2-NEXT:    callq __truncdfhf2
3194; AVX2-NEXT:    movzwl %ax, %eax
3195; AVX2-NEXT:    orl %ebx, %eax
3196; AVX2-NEXT:    shlq $32, %rax
3197; AVX2-NEXT:    orq %r14, %rax
3198; AVX2-NEXT:    vmovq %rax, %xmm0
3199; AVX2-NEXT:    addq $40, %rsp
3200; AVX2-NEXT:    popq %rbx
3201; AVX2-NEXT:    popq %r14
3202; AVX2-NEXT:    retq
3203;
3204; AVX512-LABEL: cvt_4f64_to_4i16:
3205; AVX512:       # BB#0:
3206; AVX512-NEXT:    pushq %r14
3207; AVX512-NEXT:    pushq %rbx
3208; AVX512-NEXT:    subq $40, %rsp
3209; AVX512-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3210; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3211; AVX512-NEXT:    vzeroupper
3212; AVX512-NEXT:    callq __truncdfhf2
3213; AVX512-NEXT:    movl %eax, %ebx
3214; AVX512-NEXT:    shll $16, %ebx
3215; AVX512-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3216; AVX512-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3217; AVX512-NEXT:    vzeroupper
3218; AVX512-NEXT:    callq __truncdfhf2
3219; AVX512-NEXT:    movzwl %ax, %r14d
3220; AVX512-NEXT:    orl %ebx, %r14d
3221; AVX512-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3222; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
3223; AVX512-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3224; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3225; AVX512-NEXT:    vzeroupper
3226; AVX512-NEXT:    callq __truncdfhf2
3227; AVX512-NEXT:    movl %eax, %ebx
3228; AVX512-NEXT:    shll $16, %ebx
3229; AVX512-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3230; AVX512-NEXT:    callq __truncdfhf2
3231; AVX512-NEXT:    movzwl %ax, %eax
3232; AVX512-NEXT:    orl %ebx, %eax
3233; AVX512-NEXT:    shlq $32, %rax
3234; AVX512-NEXT:    orq %r14, %rax
3235; AVX512-NEXT:    vmovq %rax, %xmm0
3236; AVX512-NEXT:    addq $40, %rsp
3237; AVX512-NEXT:    popq %rbx
3238; AVX512-NEXT:    popq %r14
3239; AVX512-NEXT:    retq
3240  %1 = fptrunc <4 x double> %a0 to <4 x half>
3241  %2 = bitcast <4 x half> %1 to <4 x i16>
3242  ret <4 x i16> %2
3243}
3244
3245define <8 x i16> @cvt_4f64_to_8i16_undef(<4 x double> %a0) nounwind {
3246; AVX1-LABEL: cvt_4f64_to_8i16_undef:
3247; AVX1:       # BB#0:
3248; AVX1-NEXT:    pushq %r14
3249; AVX1-NEXT:    pushq %rbx
3250; AVX1-NEXT:    subq $40, %rsp
3251; AVX1-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3252; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3253; AVX1-NEXT:    vzeroupper
3254; AVX1-NEXT:    callq __truncdfhf2
3255; AVX1-NEXT:    movl %eax, %ebx
3256; AVX1-NEXT:    shll $16, %ebx
3257; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3258; AVX1-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3259; AVX1-NEXT:    vzeroupper
3260; AVX1-NEXT:    callq __truncdfhf2
3261; AVX1-NEXT:    movzwl %ax, %r14d
3262; AVX1-NEXT:    orl %ebx, %r14d
3263; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3264; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3265; AVX1-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3266; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3267; AVX1-NEXT:    vzeroupper
3268; AVX1-NEXT:    callq __truncdfhf2
3269; AVX1-NEXT:    movl %eax, %ebx
3270; AVX1-NEXT:    shll $16, %ebx
3271; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3272; AVX1-NEXT:    callq __truncdfhf2
3273; AVX1-NEXT:    movzwl %ax, %eax
3274; AVX1-NEXT:    orl %ebx, %eax
3275; AVX1-NEXT:    shlq $32, %rax
3276; AVX1-NEXT:    orq %r14, %rax
3277; AVX1-NEXT:    vmovq %rax, %xmm0
3278; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
3279; AVX1-NEXT:    addq $40, %rsp
3280; AVX1-NEXT:    popq %rbx
3281; AVX1-NEXT:    popq %r14
3282; AVX1-NEXT:    retq
3283;
3284; AVX2-LABEL: cvt_4f64_to_8i16_undef:
3285; AVX2:       # BB#0:
3286; AVX2-NEXT:    pushq %r14
3287; AVX2-NEXT:    pushq %rbx
3288; AVX2-NEXT:    subq $40, %rsp
3289; AVX2-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3290; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3291; AVX2-NEXT:    vzeroupper
3292; AVX2-NEXT:    callq __truncdfhf2
3293; AVX2-NEXT:    movl %eax, %ebx
3294; AVX2-NEXT:    shll $16, %ebx
3295; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3296; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3297; AVX2-NEXT:    vzeroupper
3298; AVX2-NEXT:    callq __truncdfhf2
3299; AVX2-NEXT:    movzwl %ax, %r14d
3300; AVX2-NEXT:    orl %ebx, %r14d
3301; AVX2-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3302; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
3303; AVX2-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3304; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3305; AVX2-NEXT:    vzeroupper
3306; AVX2-NEXT:    callq __truncdfhf2
3307; AVX2-NEXT:    movl %eax, %ebx
3308; AVX2-NEXT:    shll $16, %ebx
3309; AVX2-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3310; AVX2-NEXT:    callq __truncdfhf2
3311; AVX2-NEXT:    movzwl %ax, %eax
3312; AVX2-NEXT:    orl %ebx, %eax
3313; AVX2-NEXT:    shlq $32, %rax
3314; AVX2-NEXT:    orq %r14, %rax
3315; AVX2-NEXT:    vmovq %rax, %xmm0
3316; AVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
3317; AVX2-NEXT:    addq $40, %rsp
3318; AVX2-NEXT:    popq %rbx
3319; AVX2-NEXT:    popq %r14
3320; AVX2-NEXT:    retq
3321;
3322; AVX512F-LABEL: cvt_4f64_to_8i16_undef:
3323; AVX512F:       # BB#0:
3324; AVX512F-NEXT:    pushq %r14
3325; AVX512F-NEXT:    pushq %rbx
3326; AVX512F-NEXT:    subq $40, %rsp
3327; AVX512F-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3328; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3329; AVX512F-NEXT:    vzeroupper
3330; AVX512F-NEXT:    callq __truncdfhf2
3331; AVX512F-NEXT:    movl %eax, %ebx
3332; AVX512F-NEXT:    shll $16, %ebx
3333; AVX512F-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3334; AVX512F-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3335; AVX512F-NEXT:    vzeroupper
3336; AVX512F-NEXT:    callq __truncdfhf2
3337; AVX512F-NEXT:    movzwl %ax, %r14d
3338; AVX512F-NEXT:    orl %ebx, %r14d
3339; AVX512F-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3340; AVX512F-NEXT:    vextractf128 $1, %ymm0, %xmm0
3341; AVX512F-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3342; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3343; AVX512F-NEXT:    vzeroupper
3344; AVX512F-NEXT:    callq __truncdfhf2
3345; AVX512F-NEXT:    movl %eax, %ebx
3346; AVX512F-NEXT:    shll $16, %ebx
3347; AVX512F-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3348; AVX512F-NEXT:    callq __truncdfhf2
3349; AVX512F-NEXT:    movzwl %ax, %eax
3350; AVX512F-NEXT:    orl %ebx, %eax
3351; AVX512F-NEXT:    shlq $32, %rax
3352; AVX512F-NEXT:    orq %r14, %rax
3353; AVX512F-NEXT:    vmovq %rax, %xmm0
3354; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
3355; AVX512F-NEXT:    addq $40, %rsp
3356; AVX512F-NEXT:    popq %rbx
3357; AVX512F-NEXT:    popq %r14
3358; AVX512F-NEXT:    retq
3359;
3360; AVX512VL-LABEL: cvt_4f64_to_8i16_undef:
3361; AVX512VL:       # BB#0:
3362; AVX512VL-NEXT:    pushq %r14
3363; AVX512VL-NEXT:    pushq %rbx
3364; AVX512VL-NEXT:    subq $40, %rsp
3365; AVX512VL-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3366; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3367; AVX512VL-NEXT:    vzeroupper
3368; AVX512VL-NEXT:    callq __truncdfhf2
3369; AVX512VL-NEXT:    movl %eax, %ebx
3370; AVX512VL-NEXT:    shll $16, %ebx
3371; AVX512VL-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3372; AVX512VL-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3373; AVX512VL-NEXT:    vzeroupper
3374; AVX512VL-NEXT:    callq __truncdfhf2
3375; AVX512VL-NEXT:    movzwl %ax, %r14d
3376; AVX512VL-NEXT:    orl %ebx, %r14d
3377; AVX512VL-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3378; AVX512VL-NEXT:    vextractf128 $1, %ymm0, %xmm0
3379; AVX512VL-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3380; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3381; AVX512VL-NEXT:    vzeroupper
3382; AVX512VL-NEXT:    callq __truncdfhf2
3383; AVX512VL-NEXT:    movl %eax, %ebx
3384; AVX512VL-NEXT:    shll $16, %ebx
3385; AVX512VL-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3386; AVX512VL-NEXT:    callq __truncdfhf2
3387; AVX512VL-NEXT:    movzwl %ax, %eax
3388; AVX512VL-NEXT:    orl %ebx, %eax
3389; AVX512VL-NEXT:    shlq $32, %rax
3390; AVX512VL-NEXT:    orq %r14, %rax
3391; AVX512VL-NEXT:    vmovq %rax, %xmm0
3392; AVX512VL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
3393; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3394; AVX512VL-NEXT:    addq $40, %rsp
3395; AVX512VL-NEXT:    popq %rbx
3396; AVX512VL-NEXT:    popq %r14
3397; AVX512VL-NEXT:    retq
3398  %1 = fptrunc <4 x double> %a0 to <4 x half>
3399  %2 = bitcast <4 x half> %1 to <4 x i16>
3400  %3 = shufflevector <4 x i16> %2, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
3401  ret <8 x i16> %3
3402}
3403
3404define <8 x i16> @cvt_4f64_to_8i16_zero(<4 x double> %a0) nounwind {
3405; AVX1-LABEL: cvt_4f64_to_8i16_zero:
3406; AVX1:       # BB#0:
3407; AVX1-NEXT:    pushq %r14
3408; AVX1-NEXT:    pushq %rbx
3409; AVX1-NEXT:    subq $40, %rsp
3410; AVX1-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3411; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3412; AVX1-NEXT:    vzeroupper
3413; AVX1-NEXT:    callq __truncdfhf2
3414; AVX1-NEXT:    movl %eax, %ebx
3415; AVX1-NEXT:    shll $16, %ebx
3416; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3417; AVX1-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3418; AVX1-NEXT:    vzeroupper
3419; AVX1-NEXT:    callq __truncdfhf2
3420; AVX1-NEXT:    movzwl %ax, %r14d
3421; AVX1-NEXT:    orl %ebx, %r14d
3422; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3423; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3424; AVX1-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3425; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3426; AVX1-NEXT:    vzeroupper
3427; AVX1-NEXT:    callq __truncdfhf2
3428; AVX1-NEXT:    movl %eax, %ebx
3429; AVX1-NEXT:    shll $16, %ebx
3430; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3431; AVX1-NEXT:    callq __truncdfhf2
3432; AVX1-NEXT:    movzwl %ax, %eax
3433; AVX1-NEXT:    orl %ebx, %eax
3434; AVX1-NEXT:    shlq $32, %rax
3435; AVX1-NEXT:    orq %r14, %rax
3436; AVX1-NEXT:    vmovq %rax, %xmm0
3437; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero
3438; AVX1-NEXT:    addq $40, %rsp
3439; AVX1-NEXT:    popq %rbx
3440; AVX1-NEXT:    popq %r14
3441; AVX1-NEXT:    retq
3442;
3443; AVX2-LABEL: cvt_4f64_to_8i16_zero:
3444; AVX2:       # BB#0:
3445; AVX2-NEXT:    pushq %r14
3446; AVX2-NEXT:    pushq %rbx
3447; AVX2-NEXT:    subq $40, %rsp
3448; AVX2-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3449; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3450; AVX2-NEXT:    vzeroupper
3451; AVX2-NEXT:    callq __truncdfhf2
3452; AVX2-NEXT:    movl %eax, %ebx
3453; AVX2-NEXT:    shll $16, %ebx
3454; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3455; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3456; AVX2-NEXT:    vzeroupper
3457; AVX2-NEXT:    callq __truncdfhf2
3458; AVX2-NEXT:    movzwl %ax, %r14d
3459; AVX2-NEXT:    orl %ebx, %r14d
3460; AVX2-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3461; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
3462; AVX2-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3463; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3464; AVX2-NEXT:    vzeroupper
3465; AVX2-NEXT:    callq __truncdfhf2
3466; AVX2-NEXT:    movl %eax, %ebx
3467; AVX2-NEXT:    shll $16, %ebx
3468; AVX2-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3469; AVX2-NEXT:    callq __truncdfhf2
3470; AVX2-NEXT:    movzwl %ax, %eax
3471; AVX2-NEXT:    orl %ebx, %eax
3472; AVX2-NEXT:    shlq $32, %rax
3473; AVX2-NEXT:    orq %r14, %rax
3474; AVX2-NEXT:    vmovq %rax, %xmm0
3475; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero
3476; AVX2-NEXT:    addq $40, %rsp
3477; AVX2-NEXT:    popq %rbx
3478; AVX2-NEXT:    popq %r14
3479; AVX2-NEXT:    retq
3480;
3481; AVX512F-LABEL: cvt_4f64_to_8i16_zero:
3482; AVX512F:       # BB#0:
3483; AVX512F-NEXT:    pushq %r14
3484; AVX512F-NEXT:    pushq %rbx
3485; AVX512F-NEXT:    subq $40, %rsp
3486; AVX512F-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3487; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3488; AVX512F-NEXT:    vzeroupper
3489; AVX512F-NEXT:    callq __truncdfhf2
3490; AVX512F-NEXT:    movl %eax, %ebx
3491; AVX512F-NEXT:    shll $16, %ebx
3492; AVX512F-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3493; AVX512F-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3494; AVX512F-NEXT:    vzeroupper
3495; AVX512F-NEXT:    callq __truncdfhf2
3496; AVX512F-NEXT:    movzwl %ax, %r14d
3497; AVX512F-NEXT:    orl %ebx, %r14d
3498; AVX512F-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3499; AVX512F-NEXT:    vextractf128 $1, %ymm0, %xmm0
3500; AVX512F-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3501; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3502; AVX512F-NEXT:    vzeroupper
3503; AVX512F-NEXT:    callq __truncdfhf2
3504; AVX512F-NEXT:    movl %eax, %ebx
3505; AVX512F-NEXT:    shll $16, %ebx
3506; AVX512F-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3507; AVX512F-NEXT:    callq __truncdfhf2
3508; AVX512F-NEXT:    movzwl %ax, %eax
3509; AVX512F-NEXT:    orl %ebx, %eax
3510; AVX512F-NEXT:    shlq $32, %rax
3511; AVX512F-NEXT:    orq %r14, %rax
3512; AVX512F-NEXT:    vmovq %rax, %xmm0
3513; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero
3514; AVX512F-NEXT:    addq $40, %rsp
3515; AVX512F-NEXT:    popq %rbx
3516; AVX512F-NEXT:    popq %r14
3517; AVX512F-NEXT:    retq
3518;
3519; AVX512VL-LABEL: cvt_4f64_to_8i16_zero:
3520; AVX512VL:       # BB#0:
3521; AVX512VL-NEXT:    pushq %r14
3522; AVX512VL-NEXT:    pushq %rbx
3523; AVX512VL-NEXT:    subq $40, %rsp
3524; AVX512VL-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3525; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3526; AVX512VL-NEXT:    vzeroupper
3527; AVX512VL-NEXT:    callq __truncdfhf2
3528; AVX512VL-NEXT:    movl %eax, %ebx
3529; AVX512VL-NEXT:    shll $16, %ebx
3530; AVX512VL-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3531; AVX512VL-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3532; AVX512VL-NEXT:    vzeroupper
3533; AVX512VL-NEXT:    callq __truncdfhf2
3534; AVX512VL-NEXT:    movzwl %ax, %r14d
3535; AVX512VL-NEXT:    orl %ebx, %r14d
3536; AVX512VL-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3537; AVX512VL-NEXT:    vextractf128 $1, %ymm0, %xmm0
3538; AVX512VL-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3539; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3540; AVX512VL-NEXT:    vzeroupper
3541; AVX512VL-NEXT:    callq __truncdfhf2
3542; AVX512VL-NEXT:    movl %eax, %ebx
3543; AVX512VL-NEXT:    shll $16, %ebx
3544; AVX512VL-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3545; AVX512VL-NEXT:    callq __truncdfhf2
3546; AVX512VL-NEXT:    movzwl %ax, %eax
3547; AVX512VL-NEXT:    orl %ebx, %eax
3548; AVX512VL-NEXT:    shlq $32, %rax
3549; AVX512VL-NEXT:    orq %r14, %rax
3550; AVX512VL-NEXT:    vmovq %rax, %xmm0
3551; AVX512VL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
3552; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]
3553; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
3554; AVX512VL-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
3555; AVX512VL-NEXT:    addq $40, %rsp
3556; AVX512VL-NEXT:    popq %rbx
3557; AVX512VL-NEXT:    popq %r14
3558; AVX512VL-NEXT:    retq
3559  %1 = fptrunc <4 x double> %a0 to <4 x half>
3560  %2 = bitcast <4 x half> %1 to <4 x i16>
3561  %3 = shufflevector <4 x i16> %2, <4 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
3562  ret <8 x i16> %3
3563}
3564
3565define <8 x i16> @cvt_8f64_to_8i16(<8 x double> %a0) nounwind {
3566; AVX1-LABEL: cvt_8f64_to_8i16:
3567; AVX1:       # BB#0:
3568; AVX1-NEXT:    pushq %r15
3569; AVX1-NEXT:    pushq %r14
3570; AVX1-NEXT:    pushq %rbx
3571; AVX1-NEXT:    subq $64, %rsp
3572; AVX1-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
3573; AVX1-NEXT:    vmovupd %ymm0, {{[0-9]+}}(%rsp) # 32-byte Spill
3574; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3575; AVX1-NEXT:    vzeroupper
3576; AVX1-NEXT:    callq __truncdfhf2
3577; AVX1-NEXT:    movl %eax, %ebx
3578; AVX1-NEXT:    shll $16, %ebx
3579; AVX1-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
3580; AVX1-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3581; AVX1-NEXT:    vzeroupper
3582; AVX1-NEXT:    callq __truncdfhf2
3583; AVX1-NEXT:    movzwl %ax, %r15d
3584; AVX1-NEXT:    orl %ebx, %r15d
3585; AVX1-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
3586; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3587; AVX1-NEXT:    vmovapd %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
3588; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3589; AVX1-NEXT:    vzeroupper
3590; AVX1-NEXT:    callq __truncdfhf2
3591; AVX1-NEXT:    movl %eax, %ebx
3592; AVX1-NEXT:    shll $16, %ebx
3593; AVX1-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
3594; AVX1-NEXT:    callq __truncdfhf2
3595; AVX1-NEXT:    movzwl %ax, %r14d
3596; AVX1-NEXT:    orl %ebx, %r14d
3597; AVX1-NEXT:    shlq $32, %r14
3598; AVX1-NEXT:    orq %r15, %r14
3599; AVX1-NEXT:    vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload
3600; AVX1-NEXT:    # xmm0 = mem[1,0]
3601; AVX1-NEXT:    callq __truncdfhf2
3602; AVX1-NEXT:    movl %eax, %ebx
3603; AVX1-NEXT:    shll $16, %ebx
3604; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3605; AVX1-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3606; AVX1-NEXT:    vzeroupper
3607; AVX1-NEXT:    callq __truncdfhf2
3608; AVX1-NEXT:    movzwl %ax, %r15d
3609; AVX1-NEXT:    orl %ebx, %r15d
3610; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3611; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3612; AVX1-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3613; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3614; AVX1-NEXT:    vzeroupper
3615; AVX1-NEXT:    callq __truncdfhf2
3616; AVX1-NEXT:    movl %eax, %ebx
3617; AVX1-NEXT:    shll $16, %ebx
3618; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3619; AVX1-NEXT:    callq __truncdfhf2
3620; AVX1-NEXT:    movzwl %ax, %eax
3621; AVX1-NEXT:    orl %ebx, %eax
3622; AVX1-NEXT:    shlq $32, %rax
3623; AVX1-NEXT:    orq %r15, %rax
3624; AVX1-NEXT:    vmovq %rax, %xmm0
3625; AVX1-NEXT:    vmovq %r14, %xmm1
3626; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
3627; AVX1-NEXT:    addq $64, %rsp
3628; AVX1-NEXT:    popq %rbx
3629; AVX1-NEXT:    popq %r14
3630; AVX1-NEXT:    popq %r15
3631; AVX1-NEXT:    retq
3632;
3633; AVX2-LABEL: cvt_8f64_to_8i16:
3634; AVX2:       # BB#0:
3635; AVX2-NEXT:    pushq %r15
3636; AVX2-NEXT:    pushq %r14
3637; AVX2-NEXT:    pushq %rbx
3638; AVX2-NEXT:    subq $64, %rsp
3639; AVX2-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
3640; AVX2-NEXT:    vmovupd %ymm0, {{[0-9]+}}(%rsp) # 32-byte Spill
3641; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3642; AVX2-NEXT:    vzeroupper
3643; AVX2-NEXT:    callq __truncdfhf2
3644; AVX2-NEXT:    movl %eax, %ebx
3645; AVX2-NEXT:    shll $16, %ebx
3646; AVX2-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
3647; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3648; AVX2-NEXT:    vzeroupper
3649; AVX2-NEXT:    callq __truncdfhf2
3650; AVX2-NEXT:    movzwl %ax, %r15d
3651; AVX2-NEXT:    orl %ebx, %r15d
3652; AVX2-NEXT:    vmovupd {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
3653; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
3654; AVX2-NEXT:    vmovapd %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
3655; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3656; AVX2-NEXT:    vzeroupper
3657; AVX2-NEXT:    callq __truncdfhf2
3658; AVX2-NEXT:    movl %eax, %ebx
3659; AVX2-NEXT:    shll $16, %ebx
3660; AVX2-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
3661; AVX2-NEXT:    callq __truncdfhf2
3662; AVX2-NEXT:    movzwl %ax, %r14d
3663; AVX2-NEXT:    orl %ebx, %r14d
3664; AVX2-NEXT:    shlq $32, %r14
3665; AVX2-NEXT:    orq %r15, %r14
3666; AVX2-NEXT:    vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload
3667; AVX2-NEXT:    # xmm0 = mem[1,0]
3668; AVX2-NEXT:    callq __truncdfhf2
3669; AVX2-NEXT:    movl %eax, %ebx
3670; AVX2-NEXT:    shll $16, %ebx
3671; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3672; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3673; AVX2-NEXT:    vzeroupper
3674; AVX2-NEXT:    callq __truncdfhf2
3675; AVX2-NEXT:    movzwl %ax, %r15d
3676; AVX2-NEXT:    orl %ebx, %r15d
3677; AVX2-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3678; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
3679; AVX2-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3680; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3681; AVX2-NEXT:    vzeroupper
3682; AVX2-NEXT:    callq __truncdfhf2
3683; AVX2-NEXT:    movl %eax, %ebx
3684; AVX2-NEXT:    shll $16, %ebx
3685; AVX2-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3686; AVX2-NEXT:    callq __truncdfhf2
3687; AVX2-NEXT:    movzwl %ax, %eax
3688; AVX2-NEXT:    orl %ebx, %eax
3689; AVX2-NEXT:    shlq $32, %rax
3690; AVX2-NEXT:    orq %r15, %rax
3691; AVX2-NEXT:    vmovq %rax, %xmm0
3692; AVX2-NEXT:    vmovq %r14, %xmm1
3693; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
3694; AVX2-NEXT:    addq $64, %rsp
3695; AVX2-NEXT:    popq %rbx
3696; AVX2-NEXT:    popq %r14
3697; AVX2-NEXT:    popq %r15
3698; AVX2-NEXT:    retq
3699;
3700; AVX512-LABEL: cvt_8f64_to_8i16:
3701; AVX512:       # BB#0:
3702; AVX512-NEXT:    pushq %r15
3703; AVX512-NEXT:    pushq %r14
3704; AVX512-NEXT:    pushq %rbx
3705; AVX512-NEXT:    subq $96, %rsp
3706; AVX512-NEXT:    vmovupd %zmm0, (%rsp) # 64-byte Spill
3707; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3708; AVX512-NEXT:    vzeroupper
3709; AVX512-NEXT:    callq __truncdfhf2
3710; AVX512-NEXT:    movl %eax, %ebx
3711; AVX512-NEXT:    shll $16, %ebx
3712; AVX512-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
3713; AVX512-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill>
3714; AVX512-NEXT:    vzeroupper
3715; AVX512-NEXT:    callq __truncdfhf2
3716; AVX512-NEXT:    movzwl %ax, %r15d
3717; AVX512-NEXT:    orl %ebx, %r15d
3718; AVX512-NEXT:    vmovupd (%rsp), %zmm0 # 64-byte Reload
3719; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
3720; AVX512-NEXT:    vmovapd %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
3721; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3722; AVX512-NEXT:    vzeroupper
3723; AVX512-NEXT:    callq __truncdfhf2
3724; AVX512-NEXT:    movl %eax, %ebx
3725; AVX512-NEXT:    shll $16, %ebx
3726; AVX512-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
3727; AVX512-NEXT:    callq __truncdfhf2
3728; AVX512-NEXT:    movzwl %ax, %r14d
3729; AVX512-NEXT:    orl %ebx, %r14d
3730; AVX512-NEXT:    shlq $32, %r14
3731; AVX512-NEXT:    orq %r15, %r14
3732; AVX512-NEXT:    vmovupd (%rsp), %zmm0 # 64-byte Reload
3733; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
3734; AVX512-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3735; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3736; AVX512-NEXT:    vzeroupper
3737; AVX512-NEXT:    callq __truncdfhf2
3738; AVX512-NEXT:    movl %eax, %ebx
3739; AVX512-NEXT:    shll $16, %ebx
3740; AVX512-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3741; AVX512-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3742; AVX512-NEXT:    vzeroupper
3743; AVX512-NEXT:    callq __truncdfhf2
3744; AVX512-NEXT:    movzwl %ax, %r15d
3745; AVX512-NEXT:    orl %ebx, %r15d
3746; AVX512-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3747; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
3748; AVX512-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3749; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3750; AVX512-NEXT:    vzeroupper
3751; AVX512-NEXT:    callq __truncdfhf2
3752; AVX512-NEXT:    movl %eax, %ebx
3753; AVX512-NEXT:    shll $16, %ebx
3754; AVX512-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3755; AVX512-NEXT:    callq __truncdfhf2
3756; AVX512-NEXT:    movzwl %ax, %eax
3757; AVX512-NEXT:    orl %ebx, %eax
3758; AVX512-NEXT:    shlq $32, %rax
3759; AVX512-NEXT:    orq %r15, %rax
3760; AVX512-NEXT:    vmovq %rax, %xmm0
3761; AVX512-NEXT:    vmovq %r14, %xmm1
3762; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
3763; AVX512-NEXT:    addq $96, %rsp
3764; AVX512-NEXT:    popq %rbx
3765; AVX512-NEXT:    popq %r14
3766; AVX512-NEXT:    popq %r15
3767; AVX512-NEXT:    retq
3768  %1 = fptrunc <8 x double> %a0 to <8 x half>
3769  %2 = bitcast <8 x half> %1 to <8 x i16>
3770  ret <8 x i16> %2
3771}
3772
3773;
3774; Double to Half (Store)
3775;
3776
3777define void @store_cvt_f64_to_i16(double %a0, i16* %a1) nounwind {
3778; ALL-LABEL: store_cvt_f64_to_i16:
3779; ALL:       # BB#0:
3780; ALL-NEXT:    pushq %rbx
3781; ALL-NEXT:    movq %rdi, %rbx
3782; ALL-NEXT:    callq __truncdfhf2
3783; ALL-NEXT:    movw %ax, (%rbx)
3784; ALL-NEXT:    popq %rbx
3785; ALL-NEXT:    retq
3786  %1 = fptrunc double %a0 to half
3787  %2 = bitcast half %1 to i16
3788  store i16 %2, i16* %a1
3789  ret void
3790}
3791
3792define void @store_cvt_2f64_to_2i16(<2 x double> %a0, <2 x i16>* %a1) nounwind {
3793; ALL-LABEL: store_cvt_2f64_to_2i16:
3794; ALL:       # BB#0:
3795; ALL-NEXT:    pushq %rbp
3796; ALL-NEXT:    pushq %rbx
3797; ALL-NEXT:    subq $24, %rsp
3798; ALL-NEXT:    movq %rdi, %rbx
3799; ALL-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3800; ALL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3801; ALL-NEXT:    callq __truncdfhf2
3802; ALL-NEXT:    movl %eax, %ebp
3803; ALL-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3804; ALL-NEXT:    callq __truncdfhf2
3805; ALL-NEXT:    movw %ax, (%rbx)
3806; ALL-NEXT:    movw %bp, 2(%rbx)
3807; ALL-NEXT:    addq $24, %rsp
3808; ALL-NEXT:    popq %rbx
3809; ALL-NEXT:    popq %rbp
3810; ALL-NEXT:    retq
3811  %1 = fptrunc <2 x double> %a0 to <2 x half>
3812  %2 = bitcast <2 x half> %1 to <2 x i16>
3813  store <2 x i16> %2, <2 x i16>* %a1
3814  ret void
3815}
3816
3817define void @store_cvt_4f64_to_4i16(<4 x double> %a0, <4 x i16>* %a1) nounwind {
3818; AVX1-LABEL: store_cvt_4f64_to_4i16:
3819; AVX1:       # BB#0:
3820; AVX1-NEXT:    pushq %rbp
3821; AVX1-NEXT:    pushq %r15
3822; AVX1-NEXT:    pushq %r14
3823; AVX1-NEXT:    pushq %rbx
3824; AVX1-NEXT:    subq $88, %rsp
3825; AVX1-NEXT:    movq %rdi, %rbx
3826; AVX1-NEXT:    vmovupd %ymm0, {{[0-9]+}}(%rsp) # 32-byte Spill
3827; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3828; AVX1-NEXT:    vzeroupper
3829; AVX1-NEXT:    callq __truncdfhf2
3830; AVX1-NEXT:    movl %eax, %r14d
3831; AVX1-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
3832; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3833; AVX1-NEXT:    vmovapd %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
3834; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3835; AVX1-NEXT:    vzeroupper
3836; AVX1-NEXT:    callq __truncdfhf2
3837; AVX1-NEXT:    movl %eax, %r15d
3838; AVX1-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
3839; AVX1-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3840; AVX1-NEXT:    vzeroupper
3841; AVX1-NEXT:    callq __truncdfhf2
3842; AVX1-NEXT:    movl %eax, %ebp
3843; AVX1-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
3844; AVX1-NEXT:    callq __truncdfhf2
3845; AVX1-NEXT:    movw %ax, 4(%rbx)
3846; AVX1-NEXT:    movw %bp, (%rbx)
3847; AVX1-NEXT:    movw %r15w, 6(%rbx)
3848; AVX1-NEXT:    movw %r14w, 2(%rbx)
3849; AVX1-NEXT:    addq $88, %rsp
3850; AVX1-NEXT:    popq %rbx
3851; AVX1-NEXT:    popq %r14
3852; AVX1-NEXT:    popq %r15
3853; AVX1-NEXT:    popq %rbp
3854; AVX1-NEXT:    retq
3855;
3856; AVX2-LABEL: store_cvt_4f64_to_4i16:
3857; AVX2:       # BB#0:
3858; AVX2-NEXT:    pushq %rbp
3859; AVX2-NEXT:    pushq %r15
3860; AVX2-NEXT:    pushq %r14
3861; AVX2-NEXT:    pushq %rbx
3862; AVX2-NEXT:    subq $88, %rsp
3863; AVX2-NEXT:    movq %rdi, %rbx
3864; AVX2-NEXT:    vmovupd %ymm0, {{[0-9]+}}(%rsp) # 32-byte Spill
3865; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3866; AVX2-NEXT:    vzeroupper
3867; AVX2-NEXT:    callq __truncdfhf2
3868; AVX2-NEXT:    movl %eax, %r14d
3869; AVX2-NEXT:    vmovupd {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
3870; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
3871; AVX2-NEXT:    vmovapd %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
3872; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3873; AVX2-NEXT:    vzeroupper
3874; AVX2-NEXT:    callq __truncdfhf2
3875; AVX2-NEXT:    movl %eax, %r15d
3876; AVX2-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
3877; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3878; AVX2-NEXT:    vzeroupper
3879; AVX2-NEXT:    callq __truncdfhf2
3880; AVX2-NEXT:    movl %eax, %ebp
3881; AVX2-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
3882; AVX2-NEXT:    callq __truncdfhf2
3883; AVX2-NEXT:    movw %ax, 4(%rbx)
3884; AVX2-NEXT:    movw %bp, (%rbx)
3885; AVX2-NEXT:    movw %r15w, 6(%rbx)
3886; AVX2-NEXT:    movw %r14w, 2(%rbx)
3887; AVX2-NEXT:    addq $88, %rsp
3888; AVX2-NEXT:    popq %rbx
3889; AVX2-NEXT:    popq %r14
3890; AVX2-NEXT:    popq %r15
3891; AVX2-NEXT:    popq %rbp
3892; AVX2-NEXT:    retq
3893;
3894; AVX512-LABEL: store_cvt_4f64_to_4i16:
3895; AVX512:       # BB#0:
3896; AVX512-NEXT:    pushq %rbp
3897; AVX512-NEXT:    pushq %r15
3898; AVX512-NEXT:    pushq %r14
3899; AVX512-NEXT:    pushq %rbx
3900; AVX512-NEXT:    subq $88, %rsp
3901; AVX512-NEXT:    movq %rdi, %rbx
3902; AVX512-NEXT:    vmovupd %ymm0, {{[0-9]+}}(%rsp) # 32-byte Spill
3903; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3904; AVX512-NEXT:    vzeroupper
3905; AVX512-NEXT:    callq __truncdfhf2
3906; AVX512-NEXT:    movl %eax, %r14d
3907; AVX512-NEXT:    vmovupd {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
3908; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
3909; AVX512-NEXT:    vmovapd %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
3910; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3911; AVX512-NEXT:    vzeroupper
3912; AVX512-NEXT:    callq __truncdfhf2
3913; AVX512-NEXT:    movl %eax, %r15d
3914; AVX512-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
3915; AVX512-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3916; AVX512-NEXT:    vzeroupper
3917; AVX512-NEXT:    callq __truncdfhf2
3918; AVX512-NEXT:    movl %eax, %ebp
3919; AVX512-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
3920; AVX512-NEXT:    callq __truncdfhf2
3921; AVX512-NEXT:    movw %ax, 4(%rbx)
3922; AVX512-NEXT:    movw %bp, (%rbx)
3923; AVX512-NEXT:    movw %r15w, 6(%rbx)
3924; AVX512-NEXT:    movw %r14w, 2(%rbx)
3925; AVX512-NEXT:    addq $88, %rsp
3926; AVX512-NEXT:    popq %rbx
3927; AVX512-NEXT:    popq %r14
3928; AVX512-NEXT:    popq %r15
3929; AVX512-NEXT:    popq %rbp
3930; AVX512-NEXT:    retq
3931  %1 = fptrunc <4 x double> %a0 to <4 x half>
3932  %2 = bitcast <4 x half> %1 to <4 x i16>
3933  store <4 x i16> %2, <4 x i16>* %a1
3934  ret void
3935}
3936
3937define void @store_cvt_4f64_to_8i16_undef(<4 x double> %a0, <8 x i16>* %a1) nounwind {
3938; AVX1-LABEL: store_cvt_4f64_to_8i16_undef:
3939; AVX1:       # BB#0:
3940; AVX1-NEXT:    pushq %rbp
3941; AVX1-NEXT:    pushq %r14
3942; AVX1-NEXT:    pushq %rbx
3943; AVX1-NEXT:    subq $32, %rsp
3944; AVX1-NEXT:    movq %rdi, %r14
3945; AVX1-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3946; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3947; AVX1-NEXT:    vzeroupper
3948; AVX1-NEXT:    callq __truncdfhf2
3949; AVX1-NEXT:    movl %eax, %ebp
3950; AVX1-NEXT:    shll $16, %ebp
3951; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3952; AVX1-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3953; AVX1-NEXT:    vzeroupper
3954; AVX1-NEXT:    callq __truncdfhf2
3955; AVX1-NEXT:    movzwl %ax, %ebx
3956; AVX1-NEXT:    orl %ebp, %ebx
3957; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3958; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3959; AVX1-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3960; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3961; AVX1-NEXT:    vzeroupper
3962; AVX1-NEXT:    callq __truncdfhf2
3963; AVX1-NEXT:    movl %eax, %ebp
3964; AVX1-NEXT:    shll $16, %ebp
3965; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3966; AVX1-NEXT:    callq __truncdfhf2
3967; AVX1-NEXT:    movzwl %ax, %eax
3968; AVX1-NEXT:    orl %ebp, %eax
3969; AVX1-NEXT:    shlq $32, %rax
3970; AVX1-NEXT:    orq %rbx, %rax
3971; AVX1-NEXT:    vmovq %rax, %xmm0
3972; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
3973; AVX1-NEXT:    vmovdqa %xmm0, (%r14)
3974; AVX1-NEXT:    addq $32, %rsp
3975; AVX1-NEXT:    popq %rbx
3976; AVX1-NEXT:    popq %r14
3977; AVX1-NEXT:    popq %rbp
3978; AVX1-NEXT:    retq
3979;
3980; AVX2-LABEL: store_cvt_4f64_to_8i16_undef:
3981; AVX2:       # BB#0:
3982; AVX2-NEXT:    pushq %rbp
3983; AVX2-NEXT:    pushq %r14
3984; AVX2-NEXT:    pushq %rbx
3985; AVX2-NEXT:    subq $32, %rsp
3986; AVX2-NEXT:    movq %rdi, %r14
3987; AVX2-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3988; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3989; AVX2-NEXT:    vzeroupper
3990; AVX2-NEXT:    callq __truncdfhf2
3991; AVX2-NEXT:    movl %eax, %ebp
3992; AVX2-NEXT:    shll $16, %ebp
3993; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3994; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
3995; AVX2-NEXT:    vzeroupper
3996; AVX2-NEXT:    callq __truncdfhf2
3997; AVX2-NEXT:    movzwl %ax, %ebx
3998; AVX2-NEXT:    orl %ebp, %ebx
3999; AVX2-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
4000; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
4001; AVX2-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
4002; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4003; AVX2-NEXT:    vzeroupper
4004; AVX2-NEXT:    callq __truncdfhf2
4005; AVX2-NEXT:    movl %eax, %ebp
4006; AVX2-NEXT:    shll $16, %ebp
4007; AVX2-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
4008; AVX2-NEXT:    callq __truncdfhf2
4009; AVX2-NEXT:    movzwl %ax, %eax
4010; AVX2-NEXT:    orl %ebp, %eax
4011; AVX2-NEXT:    shlq $32, %rax
4012; AVX2-NEXT:    orq %rbx, %rax
4013; AVX2-NEXT:    vmovq %rax, %xmm0
4014; AVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
4015; AVX2-NEXT:    vmovdqa %xmm0, (%r14)
4016; AVX2-NEXT:    addq $32, %rsp
4017; AVX2-NEXT:    popq %rbx
4018; AVX2-NEXT:    popq %r14
4019; AVX2-NEXT:    popq %rbp
4020; AVX2-NEXT:    retq
4021;
4022; AVX512F-LABEL: store_cvt_4f64_to_8i16_undef:
4023; AVX512F:       # BB#0:
4024; AVX512F-NEXT:    pushq %rbp
4025; AVX512F-NEXT:    pushq %r14
4026; AVX512F-NEXT:    pushq %rbx
4027; AVX512F-NEXT:    subq $32, %rsp
4028; AVX512F-NEXT:    movq %rdi, %r14
4029; AVX512F-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
4030; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4031; AVX512F-NEXT:    vzeroupper
4032; AVX512F-NEXT:    callq __truncdfhf2
4033; AVX512F-NEXT:    movl %eax, %ebp
4034; AVX512F-NEXT:    shll $16, %ebp
4035; AVX512F-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
4036; AVX512F-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
4037; AVX512F-NEXT:    vzeroupper
4038; AVX512F-NEXT:    callq __truncdfhf2
4039; AVX512F-NEXT:    movzwl %ax, %ebx
4040; AVX512F-NEXT:    orl %ebp, %ebx
4041; AVX512F-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
4042; AVX512F-NEXT:    vextractf128 $1, %ymm0, %xmm0
4043; AVX512F-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
4044; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4045; AVX512F-NEXT:    vzeroupper
4046; AVX512F-NEXT:    callq __truncdfhf2
4047; AVX512F-NEXT:    movl %eax, %ebp
4048; AVX512F-NEXT:    shll $16, %ebp
4049; AVX512F-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
4050; AVX512F-NEXT:    callq __truncdfhf2
4051; AVX512F-NEXT:    movzwl %ax, %eax
4052; AVX512F-NEXT:    orl %ebp, %eax
4053; AVX512F-NEXT:    shlq $32, %rax
4054; AVX512F-NEXT:    orq %rbx, %rax
4055; AVX512F-NEXT:    vmovq %rax, %xmm0
4056; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
4057; AVX512F-NEXT:    vmovdqa %xmm0, (%r14)
4058; AVX512F-NEXT:    addq $32, %rsp
4059; AVX512F-NEXT:    popq %rbx
4060; AVX512F-NEXT:    popq %r14
4061; AVX512F-NEXT:    popq %rbp
4062; AVX512F-NEXT:    retq
4063;
4064; AVX512VL-LABEL: store_cvt_4f64_to_8i16_undef:
4065; AVX512VL:       # BB#0:
4066; AVX512VL-NEXT:    pushq %rbp
4067; AVX512VL-NEXT:    pushq %r14
4068; AVX512VL-NEXT:    pushq %rbx
4069; AVX512VL-NEXT:    subq $32, %rsp
4070; AVX512VL-NEXT:    movq %rdi, %r14
4071; AVX512VL-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
4072; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4073; AVX512VL-NEXT:    vzeroupper
4074; AVX512VL-NEXT:    callq __truncdfhf2
4075; AVX512VL-NEXT:    movl %eax, %ebp
4076; AVX512VL-NEXT:    shll $16, %ebp
4077; AVX512VL-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
4078; AVX512VL-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
4079; AVX512VL-NEXT:    vzeroupper
4080; AVX512VL-NEXT:    callq __truncdfhf2
4081; AVX512VL-NEXT:    movzwl %ax, %ebx
4082; AVX512VL-NEXT:    orl %ebp, %ebx
4083; AVX512VL-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
4084; AVX512VL-NEXT:    vextractf128 $1, %ymm0, %xmm0
4085; AVX512VL-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
4086; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4087; AVX512VL-NEXT:    vzeroupper
4088; AVX512VL-NEXT:    callq __truncdfhf2
4089; AVX512VL-NEXT:    movl %eax, %ebp
4090; AVX512VL-NEXT:    shll $16, %ebp
4091; AVX512VL-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
4092; AVX512VL-NEXT:    callq __truncdfhf2
4093; AVX512VL-NEXT:    movzwl %ax, %eax
4094; AVX512VL-NEXT:    orl %ebp, %eax
4095; AVX512VL-NEXT:    shlq $32, %rax
4096; AVX512VL-NEXT:    orq %rbx, %rax
4097; AVX512VL-NEXT:    vmovq %rax, %xmm0
4098; AVX512VL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
4099; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
4100; AVX512VL-NEXT:    vmovdqa %xmm0, (%r14)
4101; AVX512VL-NEXT:    addq $32, %rsp
4102; AVX512VL-NEXT:    popq %rbx
4103; AVX512VL-NEXT:    popq %r14
4104; AVX512VL-NEXT:    popq %rbp
4105; AVX512VL-NEXT:    retq
4106  %1 = fptrunc <4 x double> %a0 to <4 x half>
4107  %2 = bitcast <4 x half> %1 to <4 x i16>
4108  %3 = shufflevector <4 x i16> %2, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
4109  store <8 x i16> %3, <8 x i16>* %a1
4110  ret void
4111}
4112
4113define void @store_cvt_4f64_to_8i16_zero(<4 x double> %a0, <8 x i16>* %a1) nounwind {
4114; AVX1-LABEL: store_cvt_4f64_to_8i16_zero:
4115; AVX1:       # BB#0:
4116; AVX1-NEXT:    pushq %rbp
4117; AVX1-NEXT:    pushq %r14
4118; AVX1-NEXT:    pushq %rbx
4119; AVX1-NEXT:    subq $32, %rsp
4120; AVX1-NEXT:    movq %rdi, %r14
4121; AVX1-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
4122; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4123; AVX1-NEXT:    vzeroupper
4124; AVX1-NEXT:    callq __truncdfhf2
4125; AVX1-NEXT:    movl %eax, %ebp
4126; AVX1-NEXT:    shll $16, %ebp
4127; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
4128; AVX1-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
4129; AVX1-NEXT:    vzeroupper
4130; AVX1-NEXT:    callq __truncdfhf2
4131; AVX1-NEXT:    movzwl %ax, %ebx
4132; AVX1-NEXT:    orl %ebp, %ebx
4133; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
4134; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
4135; AVX1-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
4136; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4137; AVX1-NEXT:    vzeroupper
4138; AVX1-NEXT:    callq __truncdfhf2
4139; AVX1-NEXT:    movl %eax, %ebp
4140; AVX1-NEXT:    shll $16, %ebp
4141; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
4142; AVX1-NEXT:    callq __truncdfhf2
4143; AVX1-NEXT:    movzwl %ax, %eax
4144; AVX1-NEXT:    orl %ebp, %eax
4145; AVX1-NEXT:    shlq $32, %rax
4146; AVX1-NEXT:    orq %rbx, %rax
4147; AVX1-NEXT:    vmovq %rax, %xmm0
4148; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero
4149; AVX1-NEXT:    vmovdqa %xmm0, (%r14)
4150; AVX1-NEXT:    addq $32, %rsp
4151; AVX1-NEXT:    popq %rbx
4152; AVX1-NEXT:    popq %r14
4153; AVX1-NEXT:    popq %rbp
4154; AVX1-NEXT:    retq
4155;
4156; AVX2-LABEL: store_cvt_4f64_to_8i16_zero:
4157; AVX2:       # BB#0:
4158; AVX2-NEXT:    pushq %rbp
4159; AVX2-NEXT:    pushq %r14
4160; AVX2-NEXT:    pushq %rbx
4161; AVX2-NEXT:    subq $32, %rsp
4162; AVX2-NEXT:    movq %rdi, %r14
4163; AVX2-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
4164; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4165; AVX2-NEXT:    vzeroupper
4166; AVX2-NEXT:    callq __truncdfhf2
4167; AVX2-NEXT:    movl %eax, %ebp
4168; AVX2-NEXT:    shll $16, %ebp
4169; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
4170; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
4171; AVX2-NEXT:    vzeroupper
4172; AVX2-NEXT:    callq __truncdfhf2
4173; AVX2-NEXT:    movzwl %ax, %ebx
4174; AVX2-NEXT:    orl %ebp, %ebx
4175; AVX2-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
4176; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
4177; AVX2-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
4178; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4179; AVX2-NEXT:    vzeroupper
4180; AVX2-NEXT:    callq __truncdfhf2
4181; AVX2-NEXT:    movl %eax, %ebp
4182; AVX2-NEXT:    shll $16, %ebp
4183; AVX2-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
4184; AVX2-NEXT:    callq __truncdfhf2
4185; AVX2-NEXT:    movzwl %ax, %eax
4186; AVX2-NEXT:    orl %ebp, %eax
4187; AVX2-NEXT:    shlq $32, %rax
4188; AVX2-NEXT:    orq %rbx, %rax
4189; AVX2-NEXT:    vmovq %rax, %xmm0
4190; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero
4191; AVX2-NEXT:    vmovdqa %xmm0, (%r14)
4192; AVX2-NEXT:    addq $32, %rsp
4193; AVX2-NEXT:    popq %rbx
4194; AVX2-NEXT:    popq %r14
4195; AVX2-NEXT:    popq %rbp
4196; AVX2-NEXT:    retq
4197;
4198; AVX512F-LABEL: store_cvt_4f64_to_8i16_zero:
4199; AVX512F:       # BB#0:
4200; AVX512F-NEXT:    pushq %rbp
4201; AVX512F-NEXT:    pushq %r14
4202; AVX512F-NEXT:    pushq %rbx
4203; AVX512F-NEXT:    subq $32, %rsp
4204; AVX512F-NEXT:    movq %rdi, %r14
4205; AVX512F-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
4206; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4207; AVX512F-NEXT:    vzeroupper
4208; AVX512F-NEXT:    callq __truncdfhf2
4209; AVX512F-NEXT:    movl %eax, %ebp
4210; AVX512F-NEXT:    shll $16, %ebp
4211; AVX512F-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
4212; AVX512F-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
4213; AVX512F-NEXT:    vzeroupper
4214; AVX512F-NEXT:    callq __truncdfhf2
4215; AVX512F-NEXT:    movzwl %ax, %ebx
4216; AVX512F-NEXT:    orl %ebp, %ebx
4217; AVX512F-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
4218; AVX512F-NEXT:    vextractf128 $1, %ymm0, %xmm0
4219; AVX512F-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
4220; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4221; AVX512F-NEXT:    vzeroupper
4222; AVX512F-NEXT:    callq __truncdfhf2
4223; AVX512F-NEXT:    movl %eax, %ebp
4224; AVX512F-NEXT:    shll $16, %ebp
4225; AVX512F-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
4226; AVX512F-NEXT:    callq __truncdfhf2
4227; AVX512F-NEXT:    movzwl %ax, %eax
4228; AVX512F-NEXT:    orl %ebp, %eax
4229; AVX512F-NEXT:    shlq $32, %rax
4230; AVX512F-NEXT:    orq %rbx, %rax
4231; AVX512F-NEXT:    vmovq %rax, %xmm0
4232; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero
4233; AVX512F-NEXT:    vmovdqa %xmm0, (%r14)
4234; AVX512F-NEXT:    addq $32, %rsp
4235; AVX512F-NEXT:    popq %rbx
4236; AVX512F-NEXT:    popq %r14
4237; AVX512F-NEXT:    popq %rbp
4238; AVX512F-NEXT:    retq
4239;
4240; AVX512VL-LABEL: store_cvt_4f64_to_8i16_zero:
4241; AVX512VL:       # BB#0:
4242; AVX512VL-NEXT:    pushq %rbp
4243; AVX512VL-NEXT:    pushq %r14
4244; AVX512VL-NEXT:    pushq %rbx
4245; AVX512VL-NEXT:    subq $32, %rsp
4246; AVX512VL-NEXT:    movq %rdi, %r14
4247; AVX512VL-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
4248; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4249; AVX512VL-NEXT:    vzeroupper
4250; AVX512VL-NEXT:    callq __truncdfhf2
4251; AVX512VL-NEXT:    movl %eax, %ebp
4252; AVX512VL-NEXT:    shll $16, %ebp
4253; AVX512VL-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
4254; AVX512VL-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
4255; AVX512VL-NEXT:    vzeroupper
4256; AVX512VL-NEXT:    callq __truncdfhf2
4257; AVX512VL-NEXT:    movzwl %ax, %ebx
4258; AVX512VL-NEXT:    orl %ebp, %ebx
4259; AVX512VL-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
4260; AVX512VL-NEXT:    vextractf128 $1, %ymm0, %xmm0
4261; AVX512VL-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
4262; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4263; AVX512VL-NEXT:    vzeroupper
4264; AVX512VL-NEXT:    callq __truncdfhf2
4265; AVX512VL-NEXT:    movl %eax, %ebp
4266; AVX512VL-NEXT:    shll $16, %ebp
4267; AVX512VL-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
4268; AVX512VL-NEXT:    callq __truncdfhf2
4269; AVX512VL-NEXT:    movzwl %ax, %eax
4270; AVX512VL-NEXT:    orl %ebp, %eax
4271; AVX512VL-NEXT:    shlq $32, %rax
4272; AVX512VL-NEXT:    orq %rbx, %rax
4273; AVX512VL-NEXT:    vmovq %rax, %xmm0
4274; AVX512VL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
4275; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]
4276; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
4277; AVX512VL-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
4278; AVX512VL-NEXT:    vmovdqa %xmm0, (%r14)
4279; AVX512VL-NEXT:    addq $32, %rsp
4280; AVX512VL-NEXT:    popq %rbx
4281; AVX512VL-NEXT:    popq %r14
4282; AVX512VL-NEXT:    popq %rbp
4283; AVX512VL-NEXT:    retq
4284  %1 = fptrunc <4 x double> %a0 to <4 x half>
4285  %2 = bitcast <4 x half> %1 to <4 x i16>
4286  %3 = shufflevector <4 x i16> %2, <4 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
4287  store <8 x i16> %3, <8 x i16>* %a1
4288  ret void
4289}
4290
4291define void @store_cvt_8f64_to_8i16(<8 x double> %a0, <8 x i16>* %a1) nounwind {
4292; AVX1-LABEL: store_cvt_8f64_to_8i16:
4293; AVX1:       # BB#0:
4294; AVX1-NEXT:    pushq %rbp
4295; AVX1-NEXT:    pushq %r15
4296; AVX1-NEXT:    pushq %r14
4297; AVX1-NEXT:    pushq %r13
4298; AVX1-NEXT:    pushq %r12
4299; AVX1-NEXT:    pushq %rbx
4300; AVX1-NEXT:    subq $136, %rsp
4301; AVX1-NEXT:    movq %rdi, %rbx
4302; AVX1-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp) # 32-byte Spill
4303; AVX1-NEXT:    vmovupd %ymm0, {{[0-9]+}}(%rsp) # 32-byte Spill
4304; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4305; AVX1-NEXT:    vzeroupper
4306; AVX1-NEXT:    callq __truncdfhf2
4307; AVX1-NEXT:    movw %ax, {{[0-9]+}}(%rsp) # 2-byte Spill
4308; AVX1-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
4309; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
4310; AVX1-NEXT:    vmovapd %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
4311; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4312; AVX1-NEXT:    vzeroupper
4313; AVX1-NEXT:    callq __truncdfhf2
4314; AVX1-NEXT:    movw %ax, {{[0-9]+}}(%rsp) # 2-byte Spill
4315; AVX1-NEXT:    vpermilpd $1, {{[0-9]+}}(%rsp), %xmm0 # 16-byte Folded Reload
4316; AVX1-NEXT:    # xmm0 = mem[1,0]
4317; AVX1-NEXT:    callq __truncdfhf2
4318; AVX1-NEXT:    movl %eax, %r12d
4319; AVX1-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
4320; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
4321; AVX1-NEXT:    vmovapd %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
4322; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4323; AVX1-NEXT:    vzeroupper
4324; AVX1-NEXT:    callq __truncdfhf2
4325; AVX1-NEXT:    movl %eax, %r13d
4326; AVX1-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
4327; AVX1-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
4328; AVX1-NEXT:    vzeroupper
4329; AVX1-NEXT:    callq __truncdfhf2
4330; AVX1-NEXT:    movl %eax, %ebp
4331; AVX1-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
4332; AVX1-NEXT:    callq __truncdfhf2
4333; AVX1-NEXT:    movl %eax, %r14d
4334; AVX1-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
4335; AVX1-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
4336; AVX1-NEXT:    vzeroupper
4337; AVX1-NEXT:    callq __truncdfhf2
4338; AVX1-NEXT:    movl %eax, %r15d
4339; AVX1-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
4340; AVX1-NEXT:    callq __truncdfhf2
4341; AVX1-NEXT:    movw %ax, 12(%rbx)
4342; AVX1-NEXT:    movw %r15w, 8(%rbx)
4343; AVX1-NEXT:    movw %r14w, 4(%rbx)
4344; AVX1-NEXT:    movw %bp, (%rbx)
4345; AVX1-NEXT:    movw %r13w, 14(%rbx)
4346; AVX1-NEXT:    movw %r12w, 10(%rbx)
4347; AVX1-NEXT:    movzwl {{[0-9]+}}(%rsp), %eax # 2-byte Folded Reload
4348; AVX1-NEXT:    movw %ax, 6(%rbx)
4349; AVX1-NEXT:    movzwl {{[0-9]+}}(%rsp), %eax # 2-byte Folded Reload
4350; AVX1-NEXT:    movw %ax, 2(%rbx)
4351; AVX1-NEXT:    addq $136, %rsp
4352; AVX1-NEXT:    popq %rbx
4353; AVX1-NEXT:    popq %r12
4354; AVX1-NEXT:    popq %r13
4355; AVX1-NEXT:    popq %r14
4356; AVX1-NEXT:    popq %r15
4357; AVX1-NEXT:    popq %rbp
4358; AVX1-NEXT:    retq
4359;
4360; AVX2-LABEL: store_cvt_8f64_to_8i16:
4361; AVX2:       # BB#0:
4362; AVX2-NEXT:    pushq %rbp
4363; AVX2-NEXT:    pushq %r15
4364; AVX2-NEXT:    pushq %r14
4365; AVX2-NEXT:    pushq %r13
4366; AVX2-NEXT:    pushq %r12
4367; AVX2-NEXT:    pushq %rbx
4368; AVX2-NEXT:    subq $136, %rsp
4369; AVX2-NEXT:    movq %rdi, %rbx
4370; AVX2-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp) # 32-byte Spill
4371; AVX2-NEXT:    vmovupd %ymm0, {{[0-9]+}}(%rsp) # 32-byte Spill
4372; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4373; AVX2-NEXT:    vzeroupper
4374; AVX2-NEXT:    callq __truncdfhf2
4375; AVX2-NEXT:    movw %ax, {{[0-9]+}}(%rsp) # 2-byte Spill
4376; AVX2-NEXT:    vmovupd {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
4377; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
4378; AVX2-NEXT:    vmovapd %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
4379; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4380; AVX2-NEXT:    vzeroupper
4381; AVX2-NEXT:    callq __truncdfhf2
4382; AVX2-NEXT:    movw %ax, {{[0-9]+}}(%rsp) # 2-byte Spill
4383; AVX2-NEXT:    vpermilpd $1, {{[0-9]+}}(%rsp), %xmm0 # 16-byte Folded Reload
4384; AVX2-NEXT:    # xmm0 = mem[1,0]
4385; AVX2-NEXT:    callq __truncdfhf2
4386; AVX2-NEXT:    movl %eax, %r12d
4387; AVX2-NEXT:    vmovupd {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
4388; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
4389; AVX2-NEXT:    vmovapd %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
4390; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4391; AVX2-NEXT:    vzeroupper
4392; AVX2-NEXT:    callq __truncdfhf2
4393; AVX2-NEXT:    movl %eax, %r13d
4394; AVX2-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
4395; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
4396; AVX2-NEXT:    vzeroupper
4397; AVX2-NEXT:    callq __truncdfhf2
4398; AVX2-NEXT:    movl %eax, %ebp
4399; AVX2-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
4400; AVX2-NEXT:    callq __truncdfhf2
4401; AVX2-NEXT:    movl %eax, %r14d
4402; AVX2-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
4403; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
4404; AVX2-NEXT:    vzeroupper
4405; AVX2-NEXT:    callq __truncdfhf2
4406; AVX2-NEXT:    movl %eax, %r15d
4407; AVX2-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
4408; AVX2-NEXT:    callq __truncdfhf2
4409; AVX2-NEXT:    movw %ax, 12(%rbx)
4410; AVX2-NEXT:    movw %r15w, 8(%rbx)
4411; AVX2-NEXT:    movw %r14w, 4(%rbx)
4412; AVX2-NEXT:    movw %bp, (%rbx)
4413; AVX2-NEXT:    movw %r13w, 14(%rbx)
4414; AVX2-NEXT:    movw %r12w, 10(%rbx)
4415; AVX2-NEXT:    movzwl {{[0-9]+}}(%rsp), %eax # 2-byte Folded Reload
4416; AVX2-NEXT:    movw %ax, 6(%rbx)
4417; AVX2-NEXT:    movzwl {{[0-9]+}}(%rsp), %eax # 2-byte Folded Reload
4418; AVX2-NEXT:    movw %ax, 2(%rbx)
4419; AVX2-NEXT:    addq $136, %rsp
4420; AVX2-NEXT:    popq %rbx
4421; AVX2-NEXT:    popq %r12
4422; AVX2-NEXT:    popq %r13
4423; AVX2-NEXT:    popq %r14
4424; AVX2-NEXT:    popq %r15
4425; AVX2-NEXT:    popq %rbp
4426; AVX2-NEXT:    retq
4427;
4428; AVX512-LABEL: store_cvt_8f64_to_8i16:
4429; AVX512:       # BB#0:
4430; AVX512-NEXT:    pushq %rbp
4431; AVX512-NEXT:    pushq %r15
4432; AVX512-NEXT:    pushq %r14
4433; AVX512-NEXT:    pushq %r13
4434; AVX512-NEXT:    pushq %r12
4435; AVX512-NEXT:    pushq %rbx
4436; AVX512-NEXT:    subq $200, %rsp
4437; AVX512-NEXT:    movq %rdi, %rbx
4438; AVX512-NEXT:    vmovupd %zmm0, {{[0-9]+}}(%rsp) # 64-byte Spill
4439; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4440; AVX512-NEXT:    vzeroupper
4441; AVX512-NEXT:    callq __truncdfhf2
4442; AVX512-NEXT:    movw %ax, {{[0-9]+}}(%rsp) # 2-byte Spill
4443; AVX512-NEXT:    vmovupd {{[0-9]+}}(%rsp), %zmm0 # 64-byte Reload
4444; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
4445; AVX512-NEXT:    vmovapd %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
4446; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4447; AVX512-NEXT:    vzeroupper
4448; AVX512-NEXT:    callq __truncdfhf2
4449; AVX512-NEXT:    movw %ax, {{[0-9]+}}(%rsp) # 2-byte Spill
4450; AVX512-NEXT:    vmovupd {{[0-9]+}}(%rsp), %zmm0 # 64-byte Reload
4451; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
4452; AVX512-NEXT:    vmovupd %ymm0, {{[0-9]+}}(%rsp) # 32-byte Spill
4453; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4454; AVX512-NEXT:    vzeroupper
4455; AVX512-NEXT:    callq __truncdfhf2
4456; AVX512-NEXT:    movl %eax, %r12d
4457; AVX512-NEXT:    vmovupd {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
4458; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
4459; AVX512-NEXT:    vmovapd %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
4460; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
4461; AVX512-NEXT:    vzeroupper
4462; AVX512-NEXT:    callq __truncdfhf2
4463; AVX512-NEXT:    movl %eax, %r13d
4464; AVX512-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm0 # 64-byte Reload
4465; AVX512-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill>
4466; AVX512-NEXT:    vzeroupper
4467; AVX512-NEXT:    callq __truncdfhf2
4468; AVX512-NEXT:    movl %eax, %ebp
4469; AVX512-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
4470; AVX512-NEXT:    callq __truncdfhf2
4471; AVX512-NEXT:    movl %eax, %r14d
4472; AVX512-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0 # 32-byte Reload
4473; AVX512-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
4474; AVX512-NEXT:    vzeroupper
4475; AVX512-NEXT:    callq __truncdfhf2
4476; AVX512-NEXT:    movl %eax, %r15d
4477; AVX512-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
4478; AVX512-NEXT:    callq __truncdfhf2
4479; AVX512-NEXT:    movw %ax, 12(%rbx)
4480; AVX512-NEXT:    movw %r15w, 8(%rbx)
4481; AVX512-NEXT:    movw %r14w, 4(%rbx)
4482; AVX512-NEXT:    movw %bp, (%rbx)
4483; AVX512-NEXT:    movw %r13w, 14(%rbx)
4484; AVX512-NEXT:    movw %r12w, 10(%rbx)
4485; AVX512-NEXT:    movzwl {{[0-9]+}}(%rsp), %eax # 2-byte Folded Reload
4486; AVX512-NEXT:    movw %ax, 6(%rbx)
4487; AVX512-NEXT:    movzwl {{[0-9]+}}(%rsp), %eax # 2-byte Folded Reload
4488; AVX512-NEXT:    movw %ax, 2(%rbx)
4489; AVX512-NEXT:    addq $200, %rsp
4490; AVX512-NEXT:    popq %rbx
4491; AVX512-NEXT:    popq %r12
4492; AVX512-NEXT:    popq %r13
4493; AVX512-NEXT:    popq %r14
4494; AVX512-NEXT:    popq %r15
4495; AVX512-NEXT:    popq %rbp
4496; AVX512-NEXT:    retq
4497  %1 = fptrunc <8 x double> %a0 to <8 x half>
4498  %2 = bitcast <8 x half> %1 to <8 x i16>
4499  store <8 x i16> %2, <8 x i16>* %a1
4500  ret void
4501}
4502