1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+f16c -verify-machineinstrs | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+f16c -verify-machineinstrs | FileCheck %s --check-prefixes=ALL,AVX,AVX2,AVX2-SLOW
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+f16c,+fast-variable-shuffle -verify-machineinstrs | FileCheck %s --check-prefixes=ALL,AVX,AVX2,AVX2-FAST
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -verify-machineinstrs | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+fast-variable-shuffle -verify-machineinstrs | FileCheck %s --check-prefixes=ALL,AVX,AVX512,AVX512VL
7
8;
9; Half to Float
10;
11
12define float @cvt_i16_to_f32(i16 %a0) nounwind {
13; ALL-LABEL: cvt_i16_to_f32:
14; ALL:       # %bb.0:
15; ALL-NEXT:    movswl %di, %eax
16; ALL-NEXT:    vmovd %eax, %xmm0
17; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
18; ALL-NEXT:    retq
19  %1 = bitcast i16 %a0 to half
20  %2 = fpext half %1 to float
21  ret float %2
22}
23
24define <4 x float> @cvt_4i16_to_4f32(<4 x i16> %a0) nounwind {
25; ALL-LABEL: cvt_4i16_to_4f32:
26; ALL:       # %bb.0:
27; ALL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
28; ALL-NEXT:    vmovq %xmm0, %rax
29; ALL-NEXT:    movq %rax, %rcx
30; ALL-NEXT:    movq %rax, %rdx
31; ALL-NEXT:    movswl %ax, %esi
32; ALL-NEXT:    # kill: def $eax killed $eax killed $rax
33; ALL-NEXT:    shrl $16, %eax
34; ALL-NEXT:    shrq $32, %rcx
35; ALL-NEXT:    shrq $48, %rdx
36; ALL-NEXT:    movswl %dx, %edx
37; ALL-NEXT:    vmovd %edx, %xmm0
38; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
39; ALL-NEXT:    movswl %cx, %ecx
40; ALL-NEXT:    vmovd %ecx, %xmm1
41; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
42; ALL-NEXT:    cwtl
43; ALL-NEXT:    vmovd %eax, %xmm2
44; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
45; ALL-NEXT:    vmovd %esi, %xmm3
46; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
47; ALL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
48; ALL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
49; ALL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
50; ALL-NEXT:    retq
51  %1 = bitcast <4 x i16> %a0 to <4 x half>
52  %2 = fpext <4 x half> %1 to <4 x float>
53  ret <4 x float> %2
54}
55
56define <4 x float> @cvt_8i16_to_4f32(<8 x i16> %a0) nounwind {
57; ALL-LABEL: cvt_8i16_to_4f32:
58; ALL:       # %bb.0:
59; ALL-NEXT:    vmovq %xmm0, %rax
60; ALL-NEXT:    movq %rax, %rcx
61; ALL-NEXT:    movq %rax, %rdx
62; ALL-NEXT:    movswl %ax, %esi
63; ALL-NEXT:    # kill: def $eax killed $eax killed $rax
64; ALL-NEXT:    shrl $16, %eax
65; ALL-NEXT:    shrq $32, %rcx
66; ALL-NEXT:    shrq $48, %rdx
67; ALL-NEXT:    movswl %dx, %edx
68; ALL-NEXT:    vmovd %edx, %xmm0
69; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
70; ALL-NEXT:    movswl %cx, %ecx
71; ALL-NEXT:    vmovd %ecx, %xmm1
72; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
73; ALL-NEXT:    cwtl
74; ALL-NEXT:    vmovd %eax, %xmm2
75; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
76; ALL-NEXT:    vmovd %esi, %xmm3
77; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
78; ALL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
79; ALL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
80; ALL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
81; ALL-NEXT:    retq
82  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
83  %2 = bitcast <4 x i16> %1 to <4 x half>
84  %3 = fpext <4 x half> %2 to <4 x float>
85  ret <4 x float> %3
86}
87
88define <8 x float> @cvt_8i16_to_8f32(<8 x i16> %a0) nounwind {
89; ALL-LABEL: cvt_8i16_to_8f32:
90; ALL:       # %bb.0:
91; ALL-NEXT:    vpextrq $1, %xmm0, %rdx
92; ALL-NEXT:    movq %rdx, %r8
93; ALL-NEXT:    movq %rdx, %r10
94; ALL-NEXT:    movswl %dx, %r9d
95; ALL-NEXT:    # kill: def $edx killed $edx killed $rdx
96; ALL-NEXT:    shrl $16, %edx
97; ALL-NEXT:    shrq $32, %r8
98; ALL-NEXT:    shrq $48, %r10
99; ALL-NEXT:    vmovq %xmm0, %rdi
100; ALL-NEXT:    movq %rdi, %rax
101; ALL-NEXT:    movq %rdi, %rsi
102; ALL-NEXT:    movswl %di, %ecx
103; ALL-NEXT:    # kill: def $edi killed $edi killed $rdi
104; ALL-NEXT:    shrl $16, %edi
105; ALL-NEXT:    shrq $32, %rax
106; ALL-NEXT:    shrq $48, %rsi
107; ALL-NEXT:    movswl %si, %esi
108; ALL-NEXT:    vmovd %esi, %xmm0
109; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
110; ALL-NEXT:    cwtl
111; ALL-NEXT:    vmovd %eax, %xmm1
112; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
113; ALL-NEXT:    movswl %di, %eax
114; ALL-NEXT:    vmovd %eax, %xmm2
115; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
116; ALL-NEXT:    vmovd %ecx, %xmm3
117; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
118; ALL-NEXT:    movswl %r10w, %eax
119; ALL-NEXT:    vmovd %eax, %xmm4
120; ALL-NEXT:    vcvtph2ps %xmm4, %xmm4
121; ALL-NEXT:    movswl %r8w, %eax
122; ALL-NEXT:    vmovd %eax, %xmm5
123; ALL-NEXT:    vcvtph2ps %xmm5, %xmm5
124; ALL-NEXT:    movswl %dx, %eax
125; ALL-NEXT:    vmovd %eax, %xmm6
126; ALL-NEXT:    vcvtph2ps %xmm6, %xmm6
127; ALL-NEXT:    vmovd %r9d, %xmm7
128; ALL-NEXT:    vcvtph2ps %xmm7, %xmm7
129; ALL-NEXT:    vinsertps {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[2,3]
130; ALL-NEXT:    vinsertps {{.*#+}} xmm5 = xmm6[0,1],xmm5[0],xmm6[3]
131; ALL-NEXT:    vinsertps {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[0]
132; ALL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
133; ALL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
134; ALL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
135; ALL-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
136; ALL-NEXT:    retq
137  %1 = bitcast <8 x i16> %a0 to <8 x half>
138  %2 = fpext <8 x half> %1 to <8 x float>
139  ret <8 x float> %2
140}
141
142define <16 x float> @cvt_16i16_to_16f32(<16 x i16> %a0) nounwind {
143; AVX1-LABEL: cvt_16i16_to_16f32:
144; AVX1:       # %bb.0:
145; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
146; AVX1-NEXT:    vmovq %xmm4, %rax
147; AVX1-NEXT:    movq %rax, %rcx
148; AVX1-NEXT:    shrq $48, %rcx
149; AVX1-NEXT:    movswl %cx, %ecx
150; AVX1-NEXT:    vmovd %ecx, %xmm8
151; AVX1-NEXT:    movq %rax, %rcx
152; AVX1-NEXT:    shrq $32, %rcx
153; AVX1-NEXT:    movswl %cx, %ecx
154; AVX1-NEXT:    vmovd %ecx, %xmm9
155; AVX1-NEXT:    movswl %ax, %ecx
156; AVX1-NEXT:    # kill: def $eax killed $eax killed $rax
157; AVX1-NEXT:    shrl $16, %eax
158; AVX1-NEXT:    cwtl
159; AVX1-NEXT:    vmovd %eax, %xmm10
160; AVX1-NEXT:    vpextrq $1, %xmm4, %rax
161; AVX1-NEXT:    vmovd %ecx, %xmm11
162; AVX1-NEXT:    movq %rax, %rcx
163; AVX1-NEXT:    shrq $48, %rcx
164; AVX1-NEXT:    movswl %cx, %ecx
165; AVX1-NEXT:    vmovd %ecx, %xmm12
166; AVX1-NEXT:    movq %rax, %rcx
167; AVX1-NEXT:    shrq $32, %rcx
168; AVX1-NEXT:    movswl %cx, %ecx
169; AVX1-NEXT:    vmovd %ecx, %xmm13
170; AVX1-NEXT:    movswl %ax, %ecx
171; AVX1-NEXT:    # kill: def $eax killed $eax killed $rax
172; AVX1-NEXT:    shrl $16, %eax
173; AVX1-NEXT:    cwtl
174; AVX1-NEXT:    vmovd %eax, %xmm14
175; AVX1-NEXT:    vmovq %xmm0, %rax
176; AVX1-NEXT:    vmovd %ecx, %xmm15
177; AVX1-NEXT:    movq %rax, %rcx
178; AVX1-NEXT:    shrq $48, %rcx
179; AVX1-NEXT:    movswl %cx, %ecx
180; AVX1-NEXT:    vmovd %ecx, %xmm2
181; AVX1-NEXT:    movq %rax, %rcx
182; AVX1-NEXT:    shrq $32, %rcx
183; AVX1-NEXT:    movswl %cx, %ecx
184; AVX1-NEXT:    vmovd %ecx, %xmm3
185; AVX1-NEXT:    movswl %ax, %ecx
186; AVX1-NEXT:    # kill: def $eax killed $eax killed $rax
187; AVX1-NEXT:    shrl $16, %eax
188; AVX1-NEXT:    cwtl
189; AVX1-NEXT:    vmovd %eax, %xmm4
190; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
191; AVX1-NEXT:    vmovd %ecx, %xmm0
192; AVX1-NEXT:    movq %rax, %rcx
193; AVX1-NEXT:    shrq $48, %rcx
194; AVX1-NEXT:    movswl %cx, %ecx
195; AVX1-NEXT:    vmovd %ecx, %xmm5
196; AVX1-NEXT:    movq %rax, %rcx
197; AVX1-NEXT:    shrq $32, %rcx
198; AVX1-NEXT:    movswl %cx, %ecx
199; AVX1-NEXT:    vmovd %ecx, %xmm6
200; AVX1-NEXT:    movl %eax, %ecx
201; AVX1-NEXT:    shrl $16, %ecx
202; AVX1-NEXT:    movswl %cx, %ecx
203; AVX1-NEXT:    vmovd %ecx, %xmm7
204; AVX1-NEXT:    cwtl
205; AVX1-NEXT:    vmovd %eax, %xmm1
206; AVX1-NEXT:    vcvtph2ps %xmm8, %xmm8
207; AVX1-NEXT:    vcvtph2ps %xmm9, %xmm9
208; AVX1-NEXT:    vcvtph2ps %xmm10, %xmm10
209; AVX1-NEXT:    vcvtph2ps %xmm11, %xmm11
210; AVX1-NEXT:    vcvtph2ps %xmm12, %xmm12
211; AVX1-NEXT:    vcvtph2ps %xmm13, %xmm13
212; AVX1-NEXT:    vcvtph2ps %xmm14, %xmm14
213; AVX1-NEXT:    vcvtph2ps %xmm15, %xmm15
214; AVX1-NEXT:    vcvtph2ps %xmm2, %xmm2
215; AVX1-NEXT:    vcvtph2ps %xmm3, %xmm3
216; AVX1-NEXT:    vcvtph2ps %xmm4, %xmm4
217; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
218; AVX1-NEXT:    vcvtph2ps %xmm5, %xmm5
219; AVX1-NEXT:    vcvtph2ps %xmm6, %xmm6
220; AVX1-NEXT:    vcvtph2ps %xmm7, %xmm7
221; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
222; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[2,3]
223; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm6[0],xmm1[3]
224; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[0]
225; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[2,3]
226; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm3[0],xmm0[3]
227; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[0]
228; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
229; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm15[0],xmm14[0],xmm15[2,3]
230; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm13[0],xmm1[3]
231; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm12[0]
232; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm11[0],xmm10[0],xmm11[2,3]
233; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
234; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
235; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
236; AVX1-NEXT:    retq
237;
238; AVX2-LABEL: cvt_16i16_to_16f32:
239; AVX2:       # %bb.0:
240; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
241; AVX2-NEXT:    vmovq %xmm4, %rax
242; AVX2-NEXT:    movq %rax, %rcx
243; AVX2-NEXT:    shrq $48, %rcx
244; AVX2-NEXT:    movswl %cx, %ecx
245; AVX2-NEXT:    vmovd %ecx, %xmm8
246; AVX2-NEXT:    movq %rax, %rcx
247; AVX2-NEXT:    shrq $32, %rcx
248; AVX2-NEXT:    movswl %cx, %ecx
249; AVX2-NEXT:    vmovd %ecx, %xmm9
250; AVX2-NEXT:    movswl %ax, %ecx
251; AVX2-NEXT:    # kill: def $eax killed $eax killed $rax
252; AVX2-NEXT:    shrl $16, %eax
253; AVX2-NEXT:    cwtl
254; AVX2-NEXT:    vmovd %eax, %xmm10
255; AVX2-NEXT:    vpextrq $1, %xmm4, %rax
256; AVX2-NEXT:    vmovd %ecx, %xmm11
257; AVX2-NEXT:    movq %rax, %rcx
258; AVX2-NEXT:    shrq $48, %rcx
259; AVX2-NEXT:    movswl %cx, %ecx
260; AVX2-NEXT:    vmovd %ecx, %xmm12
261; AVX2-NEXT:    movq %rax, %rcx
262; AVX2-NEXT:    shrq $32, %rcx
263; AVX2-NEXT:    movswl %cx, %ecx
264; AVX2-NEXT:    vmovd %ecx, %xmm13
265; AVX2-NEXT:    movswl %ax, %ecx
266; AVX2-NEXT:    # kill: def $eax killed $eax killed $rax
267; AVX2-NEXT:    shrl $16, %eax
268; AVX2-NEXT:    cwtl
269; AVX2-NEXT:    vmovd %eax, %xmm14
270; AVX2-NEXT:    vmovq %xmm0, %rax
271; AVX2-NEXT:    vmovd %ecx, %xmm15
272; AVX2-NEXT:    movq %rax, %rcx
273; AVX2-NEXT:    shrq $48, %rcx
274; AVX2-NEXT:    movswl %cx, %ecx
275; AVX2-NEXT:    vmovd %ecx, %xmm2
276; AVX2-NEXT:    movq %rax, %rcx
277; AVX2-NEXT:    shrq $32, %rcx
278; AVX2-NEXT:    movswl %cx, %ecx
279; AVX2-NEXT:    vmovd %ecx, %xmm3
280; AVX2-NEXT:    movswl %ax, %ecx
281; AVX2-NEXT:    # kill: def $eax killed $eax killed $rax
282; AVX2-NEXT:    shrl $16, %eax
283; AVX2-NEXT:    cwtl
284; AVX2-NEXT:    vmovd %eax, %xmm4
285; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
286; AVX2-NEXT:    vmovd %ecx, %xmm0
287; AVX2-NEXT:    movq %rax, %rcx
288; AVX2-NEXT:    shrq $48, %rcx
289; AVX2-NEXT:    movswl %cx, %ecx
290; AVX2-NEXT:    vmovd %ecx, %xmm5
291; AVX2-NEXT:    movq %rax, %rcx
292; AVX2-NEXT:    shrq $32, %rcx
293; AVX2-NEXT:    movswl %cx, %ecx
294; AVX2-NEXT:    vmovd %ecx, %xmm6
295; AVX2-NEXT:    movl %eax, %ecx
296; AVX2-NEXT:    shrl $16, %ecx
297; AVX2-NEXT:    movswl %cx, %ecx
298; AVX2-NEXT:    vmovd %ecx, %xmm7
299; AVX2-NEXT:    cwtl
300; AVX2-NEXT:    vmovd %eax, %xmm1
301; AVX2-NEXT:    vcvtph2ps %xmm8, %xmm8
302; AVX2-NEXT:    vcvtph2ps %xmm9, %xmm9
303; AVX2-NEXT:    vcvtph2ps %xmm10, %xmm10
304; AVX2-NEXT:    vcvtph2ps %xmm11, %xmm11
305; AVX2-NEXT:    vcvtph2ps %xmm12, %xmm12
306; AVX2-NEXT:    vcvtph2ps %xmm13, %xmm13
307; AVX2-NEXT:    vcvtph2ps %xmm14, %xmm14
308; AVX2-NEXT:    vcvtph2ps %xmm15, %xmm15
309; AVX2-NEXT:    vcvtph2ps %xmm2, %xmm2
310; AVX2-NEXT:    vcvtph2ps %xmm3, %xmm3
311; AVX2-NEXT:    vcvtph2ps %xmm4, %xmm4
312; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
313; AVX2-NEXT:    vcvtph2ps %xmm5, %xmm5
314; AVX2-NEXT:    vcvtph2ps %xmm6, %xmm6
315; AVX2-NEXT:    vcvtph2ps %xmm7, %xmm7
316; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
317; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[2,3]
318; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm6[0],xmm1[3]
319; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[0]
320; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[2,3]
321; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm3[0],xmm0[3]
322; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[0]
323; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
324; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm15[0],xmm14[0],xmm15[2,3]
325; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm13[0],xmm1[3]
326; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm12[0]
327; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm11[0],xmm10[0],xmm11[2,3]
328; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
329; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
330; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
331; AVX2-NEXT:    retq
332;
333; AVX512F-LABEL: cvt_16i16_to_16f32:
334; AVX512F:       # %bb.0:
335; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm10
336; AVX512F-NEXT:    vmovq %xmm0, %rax
337; AVX512F-NEXT:    movq %rax, %rcx
338; AVX512F-NEXT:    shrq $48, %rcx
339; AVX512F-NEXT:    movswl %cx, %ecx
340; AVX512F-NEXT:    vmovd %ecx, %xmm8
341; AVX512F-NEXT:    movq %rax, %rcx
342; AVX512F-NEXT:    shrq $32, %rcx
343; AVX512F-NEXT:    movswl %cx, %ecx
344; AVX512F-NEXT:    vmovd %ecx, %xmm9
345; AVX512F-NEXT:    movswl %ax, %ecx
346; AVX512F-NEXT:    # kill: def $eax killed $eax killed $rax
347; AVX512F-NEXT:    shrl $16, %eax
348; AVX512F-NEXT:    cwtl
349; AVX512F-NEXT:    vmovd %eax, %xmm11
350; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
351; AVX512F-NEXT:    vmovd %ecx, %xmm12
352; AVX512F-NEXT:    movq %rax, %rcx
353; AVX512F-NEXT:    shrq $48, %rcx
354; AVX512F-NEXT:    movswl %cx, %ecx
355; AVX512F-NEXT:    vmovd %ecx, %xmm13
356; AVX512F-NEXT:    movq %rax, %rcx
357; AVX512F-NEXT:    shrq $32, %rcx
358; AVX512F-NEXT:    movswl %cx, %ecx
359; AVX512F-NEXT:    vmovd %ecx, %xmm14
360; AVX512F-NEXT:    movswl %ax, %ecx
361; AVX512F-NEXT:    # kill: def $eax killed $eax killed $rax
362; AVX512F-NEXT:    shrl $16, %eax
363; AVX512F-NEXT:    cwtl
364; AVX512F-NEXT:    vmovd %eax, %xmm15
365; AVX512F-NEXT:    vmovq %xmm10, %rax
366; AVX512F-NEXT:    vmovd %ecx, %xmm2
367; AVX512F-NEXT:    movq %rax, %rcx
368; AVX512F-NEXT:    shrq $48, %rcx
369; AVX512F-NEXT:    movswl %cx, %ecx
370; AVX512F-NEXT:    vmovd %ecx, %xmm3
371; AVX512F-NEXT:    movq %rax, %rcx
372; AVX512F-NEXT:    shrq $32, %rcx
373; AVX512F-NEXT:    movswl %cx, %ecx
374; AVX512F-NEXT:    vmovd %ecx, %xmm1
375; AVX512F-NEXT:    movswl %ax, %ecx
376; AVX512F-NEXT:    # kill: def $eax killed $eax killed $rax
377; AVX512F-NEXT:    shrl $16, %eax
378; AVX512F-NEXT:    cwtl
379; AVX512F-NEXT:    vmovd %eax, %xmm4
380; AVX512F-NEXT:    vpextrq $1, %xmm10, %rax
381; AVX512F-NEXT:    vmovd %ecx, %xmm10
382; AVX512F-NEXT:    movq %rax, %rcx
383; AVX512F-NEXT:    shrq $48, %rcx
384; AVX512F-NEXT:    movswl %cx, %ecx
385; AVX512F-NEXT:    vmovd %ecx, %xmm5
386; AVX512F-NEXT:    movq %rax, %rcx
387; AVX512F-NEXT:    shrq $32, %rcx
388; AVX512F-NEXT:    movswl %cx, %ecx
389; AVX512F-NEXT:    vmovd %ecx, %xmm6
390; AVX512F-NEXT:    movl %eax, %ecx
391; AVX512F-NEXT:    shrl $16, %ecx
392; AVX512F-NEXT:    movswl %cx, %ecx
393; AVX512F-NEXT:    vmovd %ecx, %xmm7
394; AVX512F-NEXT:    cwtl
395; AVX512F-NEXT:    vmovd %eax, %xmm0
396; AVX512F-NEXT:    vcvtph2ps %xmm8, %xmm8
397; AVX512F-NEXT:    vcvtph2ps %xmm9, %xmm9
398; AVX512F-NEXT:    vcvtph2ps %xmm11, %xmm11
399; AVX512F-NEXT:    vcvtph2ps %xmm12, %xmm12
400; AVX512F-NEXT:    vcvtph2ps %xmm13, %xmm13
401; AVX512F-NEXT:    vcvtph2ps %xmm14, %xmm14
402; AVX512F-NEXT:    vcvtph2ps %xmm15, %xmm15
403; AVX512F-NEXT:    vcvtph2ps %xmm2, %xmm2
404; AVX512F-NEXT:    vcvtph2ps %xmm3, %xmm3
405; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
406; AVX512F-NEXT:    vcvtph2ps %xmm4, %xmm4
407; AVX512F-NEXT:    vcvtph2ps %xmm10, %xmm10
408; AVX512F-NEXT:    vcvtph2ps %xmm5, %xmm5
409; AVX512F-NEXT:    vcvtph2ps %xmm6, %xmm6
410; AVX512F-NEXT:    vcvtph2ps %xmm7, %xmm7
411; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
412; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[2,3]
413; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm6[0],xmm0[3]
414; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm5[0]
415; AVX512F-NEXT:    vinsertps {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[2,3]
416; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm4[0,1],xmm1[0],xmm4[3]
417; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[0]
418; AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
419; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm15[0],xmm2[2,3]
420; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm14[0],xmm1[3]
421; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm13[0]
422; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm12[0],xmm11[0],xmm12[2,3]
423; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
424; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
425; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
426; AVX512F-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
427; AVX512F-NEXT:    retq
428;
429; AVX512VL-LABEL: cvt_16i16_to_16f32:
430; AVX512VL:       # %bb.0:
431; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm10
432; AVX512VL-NEXT:    vmovq %xmm0, %rax
433; AVX512VL-NEXT:    movq %rax, %rcx
434; AVX512VL-NEXT:    shrq $48, %rcx
435; AVX512VL-NEXT:    movswl %cx, %ecx
436; AVX512VL-NEXT:    vmovd %ecx, %xmm8
437; AVX512VL-NEXT:    movq %rax, %rcx
438; AVX512VL-NEXT:    shrq $32, %rcx
439; AVX512VL-NEXT:    movswl %cx, %ecx
440; AVX512VL-NEXT:    vmovd %ecx, %xmm9
441; AVX512VL-NEXT:    movswl %ax, %ecx
442; AVX512VL-NEXT:    # kill: def $eax killed $eax killed $rax
443; AVX512VL-NEXT:    shrl $16, %eax
444; AVX512VL-NEXT:    cwtl
445; AVX512VL-NEXT:    vmovd %eax, %xmm11
446; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
447; AVX512VL-NEXT:    vmovd %ecx, %xmm12
448; AVX512VL-NEXT:    movq %rax, %rcx
449; AVX512VL-NEXT:    shrq $48, %rcx
450; AVX512VL-NEXT:    movswl %cx, %ecx
451; AVX512VL-NEXT:    vmovd %ecx, %xmm13
452; AVX512VL-NEXT:    movq %rax, %rcx
453; AVX512VL-NEXT:    shrq $32, %rcx
454; AVX512VL-NEXT:    movswl %cx, %ecx
455; AVX512VL-NEXT:    vmovd %ecx, %xmm14
456; AVX512VL-NEXT:    movswl %ax, %ecx
457; AVX512VL-NEXT:    # kill: def $eax killed $eax killed $rax
458; AVX512VL-NEXT:    shrl $16, %eax
459; AVX512VL-NEXT:    cwtl
460; AVX512VL-NEXT:    vmovd %eax, %xmm15
461; AVX512VL-NEXT:    vmovq %xmm10, %rax
462; AVX512VL-NEXT:    vmovd %ecx, %xmm16
463; AVX512VL-NEXT:    movq %rax, %rcx
464; AVX512VL-NEXT:    shrq $48, %rcx
465; AVX512VL-NEXT:    movswl %cx, %ecx
466; AVX512VL-NEXT:    vmovd %ecx, %xmm17
467; AVX512VL-NEXT:    movq %rax, %rcx
468; AVX512VL-NEXT:    shrq $32, %rcx
469; AVX512VL-NEXT:    movswl %cx, %ecx
470; AVX512VL-NEXT:    vmovd %ecx, %xmm18
471; AVX512VL-NEXT:    movswl %ax, %ecx
472; AVX512VL-NEXT:    # kill: def $eax killed $eax killed $rax
473; AVX512VL-NEXT:    shrl $16, %eax
474; AVX512VL-NEXT:    cwtl
475; AVX512VL-NEXT:    vmovd %eax, %xmm19
476; AVX512VL-NEXT:    vpextrq $1, %xmm10, %rax
477; AVX512VL-NEXT:    vmovd %ecx, %xmm10
478; AVX512VL-NEXT:    movq %rax, %rcx
479; AVX512VL-NEXT:    shrq $48, %rcx
480; AVX512VL-NEXT:    movswl %cx, %ecx
481; AVX512VL-NEXT:    vmovd %ecx, %xmm20
482; AVX512VL-NEXT:    movq %rax, %rcx
483; AVX512VL-NEXT:    shrq $32, %rcx
484; AVX512VL-NEXT:    movswl %cx, %ecx
485; AVX512VL-NEXT:    vmovd %ecx, %xmm21
486; AVX512VL-NEXT:    movl %eax, %ecx
487; AVX512VL-NEXT:    shrl $16, %ecx
488; AVX512VL-NEXT:    movswl %cx, %ecx
489; AVX512VL-NEXT:    vmovd %ecx, %xmm22
490; AVX512VL-NEXT:    cwtl
491; AVX512VL-NEXT:    vmovd %eax, %xmm2
492; AVX512VL-NEXT:    vcvtph2ps %xmm8, %xmm8
493; AVX512VL-NEXT:    vcvtph2ps %xmm9, %xmm9
494; AVX512VL-NEXT:    vcvtph2ps %xmm11, %xmm11
495; AVX512VL-NEXT:    vcvtph2ps %xmm12, %xmm12
496; AVX512VL-NEXT:    vcvtph2ps %xmm13, %xmm13
497; AVX512VL-NEXT:    vcvtph2ps %xmm14, %xmm14
498; AVX512VL-NEXT:    vcvtph2ps %xmm15, %xmm15
499; AVX512VL-NEXT:    vcvtph2ps %xmm16, %xmm16
500; AVX512VL-NEXT:    vcvtph2ps %xmm17, %xmm4
501; AVX512VL-NEXT:    vcvtph2ps %xmm18, %xmm0
502; AVX512VL-NEXT:    vcvtph2ps %xmm19, %xmm5
503; AVX512VL-NEXT:    vcvtph2ps %xmm10, %xmm7
504; AVX512VL-NEXT:    vcvtph2ps %xmm20, %xmm3
505; AVX512VL-NEXT:    vcvtph2ps %xmm21, %xmm6
506; AVX512VL-NEXT:    vcvtph2ps %xmm22, %xmm1
507; AVX512VL-NEXT:    vcvtph2ps %xmm2, %xmm2
508; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
509; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm6[0],xmm1[3]
510; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[0]
511; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm7[0],xmm5[0],xmm7[2,3]
512; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1],xmm0[0],xmm2[3]
513; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm4[0]
514; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
515; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm16[0],xmm15[0],xmm16[2,3]
516; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm14[0],xmm1[3]
517; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm13[0]
518; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm12[0],xmm11[0],xmm12[2,3]
519; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
520; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
521; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
522; AVX512VL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
523; AVX512VL-NEXT:    retq
524  %1 = bitcast <16 x i16> %a0 to <16 x half>
525  %2 = fpext <16 x half> %1 to <16 x float>
526  ret <16 x float> %2
527}
528
529;
530; Half to Float (Load)
531;
532
533define float @load_cvt_i16_to_f32(i16* %a0) nounwind {
534; ALL-LABEL: load_cvt_i16_to_f32:
535; ALL:       # %bb.0:
536; ALL-NEXT:    movswl (%rdi), %eax
537; ALL-NEXT:    vmovd %eax, %xmm0
538; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
539; ALL-NEXT:    retq
540  %1 = load i16, i16* %a0
541  %2 = bitcast i16 %1 to half
542  %3 = fpext half %2 to float
543  ret float %3
544}
545
546define <4 x float> @load_cvt_4i16_to_4f32(<4 x i16>* %a0) nounwind {
547; ALL-LABEL: load_cvt_4i16_to_4f32:
548; ALL:       # %bb.0:
549; ALL-NEXT:    movswl 6(%rdi), %eax
550; ALL-NEXT:    vmovd %eax, %xmm0
551; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
552; ALL-NEXT:    movswl 4(%rdi), %eax
553; ALL-NEXT:    vmovd %eax, %xmm1
554; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
555; ALL-NEXT:    movswl (%rdi), %eax
556; ALL-NEXT:    vmovd %eax, %xmm2
557; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
558; ALL-NEXT:    movswl 2(%rdi), %eax
559; ALL-NEXT:    vmovd %eax, %xmm3
560; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
561; ALL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
562; ALL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
563; ALL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
564; ALL-NEXT:    retq
565  %1 = load <4 x i16>, <4 x i16>* %a0
566  %2 = bitcast <4 x i16> %1 to <4 x half>
567  %3 = fpext <4 x half> %2 to <4 x float>
568  ret <4 x float> %3
569}
570
571define <4 x float> @load_cvt_8i16_to_4f32(<8 x i16>* %a0) nounwind {
572; ALL-LABEL: load_cvt_8i16_to_4f32:
573; ALL:       # %bb.0:
574; ALL-NEXT:    movq (%rdi), %rax
575; ALL-NEXT:    movq %rax, %rcx
576; ALL-NEXT:    movq %rax, %rdx
577; ALL-NEXT:    movswl %ax, %esi
578; ALL-NEXT:    # kill: def $eax killed $eax killed $rax
579; ALL-NEXT:    shrl $16, %eax
580; ALL-NEXT:    shrq $32, %rcx
581; ALL-NEXT:    shrq $48, %rdx
582; ALL-NEXT:    movswl %dx, %edx
583; ALL-NEXT:    vmovd %edx, %xmm0
584; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
585; ALL-NEXT:    movswl %cx, %ecx
586; ALL-NEXT:    vmovd %ecx, %xmm1
587; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
588; ALL-NEXT:    cwtl
589; ALL-NEXT:    vmovd %eax, %xmm2
590; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
591; ALL-NEXT:    vmovd %esi, %xmm3
592; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
593; ALL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
594; ALL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
595; ALL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
596; ALL-NEXT:    retq
597  %1 = load <8 x i16>, <8 x i16>* %a0
598  %2 = shufflevector <8 x i16> %1, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
599  %3 = bitcast <4 x i16> %2 to <4 x half>
600  %4 = fpext <4 x half> %3 to <4 x float>
601  ret <4 x float> %4
602}
603
604define <8 x float> @load_cvt_8i16_to_8f32(<8 x i16>* %a0) nounwind {
605; ALL-LABEL: load_cvt_8i16_to_8f32:
606; ALL:       # %bb.0:
607; ALL-NEXT:    movswl 6(%rdi), %eax
608; ALL-NEXT:    vmovd %eax, %xmm0
609; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
610; ALL-NEXT:    movswl 4(%rdi), %eax
611; ALL-NEXT:    vmovd %eax, %xmm1
612; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
613; ALL-NEXT:    movswl (%rdi), %eax
614; ALL-NEXT:    vmovd %eax, %xmm2
615; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
616; ALL-NEXT:    movswl 2(%rdi), %eax
617; ALL-NEXT:    vmovd %eax, %xmm3
618; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
619; ALL-NEXT:    movswl 14(%rdi), %eax
620; ALL-NEXT:    vmovd %eax, %xmm4
621; ALL-NEXT:    vcvtph2ps %xmm4, %xmm4
622; ALL-NEXT:    movswl 12(%rdi), %eax
623; ALL-NEXT:    vmovd %eax, %xmm5
624; ALL-NEXT:    vcvtph2ps %xmm5, %xmm5
625; ALL-NEXT:    movswl 8(%rdi), %eax
626; ALL-NEXT:    vmovd %eax, %xmm6
627; ALL-NEXT:    vcvtph2ps %xmm6, %xmm6
628; ALL-NEXT:    movswl 10(%rdi), %eax
629; ALL-NEXT:    vmovd %eax, %xmm7
630; ALL-NEXT:    vcvtph2ps %xmm7, %xmm7
631; ALL-NEXT:    vinsertps {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[2,3]
632; ALL-NEXT:    vinsertps {{.*#+}} xmm5 = xmm6[0,1],xmm5[0],xmm6[3]
633; ALL-NEXT:    vinsertps {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[0]
634; ALL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
635; ALL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
636; ALL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
637; ALL-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
638; ALL-NEXT:    retq
639  %1 = load <8 x i16>, <8 x i16>* %a0
640  %2 = bitcast <8 x i16> %1 to <8 x half>
641  %3 = fpext <8 x half> %2 to <8 x float>
642  ret <8 x float> %3
643}
644
645define <16 x float> @load_cvt_16i16_to_16f32(<16 x i16>* %a0) nounwind {
646; AVX1-LABEL: load_cvt_16i16_to_16f32:
647; AVX1:       # %bb.0:
648; AVX1-NEXT:    movswl 22(%rdi), %eax
649; AVX1-NEXT:    vmovd %eax, %xmm0
650; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm8
651; AVX1-NEXT:    movswl 20(%rdi), %eax
652; AVX1-NEXT:    vmovd %eax, %xmm0
653; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm9
654; AVX1-NEXT:    movswl 16(%rdi), %eax
655; AVX1-NEXT:    vmovd %eax, %xmm0
656; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm10
657; AVX1-NEXT:    movswl 18(%rdi), %eax
658; AVX1-NEXT:    vmovd %eax, %xmm0
659; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm11
660; AVX1-NEXT:    movswl 30(%rdi), %eax
661; AVX1-NEXT:    vmovd %eax, %xmm0
662; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm12
663; AVX1-NEXT:    movswl 28(%rdi), %eax
664; AVX1-NEXT:    vmovd %eax, %xmm0
665; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm13
666; AVX1-NEXT:    movswl 24(%rdi), %eax
667; AVX1-NEXT:    vmovd %eax, %xmm0
668; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm14
669; AVX1-NEXT:    movswl 26(%rdi), %eax
670; AVX1-NEXT:    vmovd %eax, %xmm0
671; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm15
672; AVX1-NEXT:    movswl 6(%rdi), %eax
673; AVX1-NEXT:    vmovd %eax, %xmm0
674; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
675; AVX1-NEXT:    movswl 4(%rdi), %eax
676; AVX1-NEXT:    vmovd %eax, %xmm2
677; AVX1-NEXT:    vcvtph2ps %xmm2, %xmm2
678; AVX1-NEXT:    movswl (%rdi), %eax
679; AVX1-NEXT:    vmovd %eax, %xmm3
680; AVX1-NEXT:    vcvtph2ps %xmm3, %xmm3
681; AVX1-NEXT:    movswl 2(%rdi), %eax
682; AVX1-NEXT:    vmovd %eax, %xmm4
683; AVX1-NEXT:    vcvtph2ps %xmm4, %xmm4
684; AVX1-NEXT:    movswl 14(%rdi), %eax
685; AVX1-NEXT:    vmovd %eax, %xmm5
686; AVX1-NEXT:    vcvtph2ps %xmm5, %xmm5
687; AVX1-NEXT:    movswl 12(%rdi), %eax
688; AVX1-NEXT:    vmovd %eax, %xmm6
689; AVX1-NEXT:    vcvtph2ps %xmm6, %xmm6
690; AVX1-NEXT:    movswl 8(%rdi), %eax
691; AVX1-NEXT:    vmovd %eax, %xmm7
692; AVX1-NEXT:    vcvtph2ps %xmm7, %xmm7
693; AVX1-NEXT:    movswl 10(%rdi), %eax
694; AVX1-NEXT:    vmovd %eax, %xmm1
695; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
696; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm7[0],xmm1[0],xmm7[2,3]
697; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm6[0],xmm1[3]
698; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[0]
699; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[2,3]
700; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0,1],xmm2[0],xmm3[3]
701; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
702; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
703; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm14[0],xmm15[0],xmm14[2,3]
704; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm13[0],xmm1[3]
705; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm12[0]
706; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm10[0],xmm11[0],xmm10[2,3]
707; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
708; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
709; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
710; AVX1-NEXT:    retq
711;
712; AVX2-LABEL: load_cvt_16i16_to_16f32:
713; AVX2:       # %bb.0:
714; AVX2-NEXT:    movswl 22(%rdi), %eax
715; AVX2-NEXT:    vmovd %eax, %xmm0
716; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm8
717; AVX2-NEXT:    movswl 20(%rdi), %eax
718; AVX2-NEXT:    vmovd %eax, %xmm0
719; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm9
720; AVX2-NEXT:    movswl 16(%rdi), %eax
721; AVX2-NEXT:    vmovd %eax, %xmm0
722; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm10
723; AVX2-NEXT:    movswl 18(%rdi), %eax
724; AVX2-NEXT:    vmovd %eax, %xmm0
725; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm11
726; AVX2-NEXT:    movswl 30(%rdi), %eax
727; AVX2-NEXT:    vmovd %eax, %xmm0
728; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm12
729; AVX2-NEXT:    movswl 28(%rdi), %eax
730; AVX2-NEXT:    vmovd %eax, %xmm0
731; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm13
732; AVX2-NEXT:    movswl 24(%rdi), %eax
733; AVX2-NEXT:    vmovd %eax, %xmm0
734; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm14
735; AVX2-NEXT:    movswl 26(%rdi), %eax
736; AVX2-NEXT:    vmovd %eax, %xmm0
737; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm15
738; AVX2-NEXT:    movswl 6(%rdi), %eax
739; AVX2-NEXT:    vmovd %eax, %xmm0
740; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
741; AVX2-NEXT:    movswl 4(%rdi), %eax
742; AVX2-NEXT:    vmovd %eax, %xmm2
743; AVX2-NEXT:    vcvtph2ps %xmm2, %xmm2
744; AVX2-NEXT:    movswl (%rdi), %eax
745; AVX2-NEXT:    vmovd %eax, %xmm3
746; AVX2-NEXT:    vcvtph2ps %xmm3, %xmm3
747; AVX2-NEXT:    movswl 2(%rdi), %eax
748; AVX2-NEXT:    vmovd %eax, %xmm4
749; AVX2-NEXT:    vcvtph2ps %xmm4, %xmm4
750; AVX2-NEXT:    movswl 14(%rdi), %eax
751; AVX2-NEXT:    vmovd %eax, %xmm5
752; AVX2-NEXT:    vcvtph2ps %xmm5, %xmm5
753; AVX2-NEXT:    movswl 12(%rdi), %eax
754; AVX2-NEXT:    vmovd %eax, %xmm6
755; AVX2-NEXT:    vcvtph2ps %xmm6, %xmm6
756; AVX2-NEXT:    movswl 8(%rdi), %eax
757; AVX2-NEXT:    vmovd %eax, %xmm7
758; AVX2-NEXT:    vcvtph2ps %xmm7, %xmm7
759; AVX2-NEXT:    movswl 10(%rdi), %eax
760; AVX2-NEXT:    vmovd %eax, %xmm1
761; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
762; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm7[0],xmm1[0],xmm7[2,3]
763; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm6[0],xmm1[3]
764; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[0]
765; AVX2-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[2,3]
766; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0,1],xmm2[0],xmm3[3]
767; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
768; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
769; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm14[0],xmm15[0],xmm14[2,3]
770; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm13[0],xmm1[3]
771; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm12[0]
772; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm10[0],xmm11[0],xmm10[2,3]
773; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
774; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
775; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
776; AVX2-NEXT:    retq
777;
778; AVX512F-LABEL: load_cvt_16i16_to_16f32:
779; AVX512F:       # %bb.0:
780; AVX512F-NEXT:    movswl 6(%rdi), %eax
781; AVX512F-NEXT:    vmovd %eax, %xmm0
782; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm8
783; AVX512F-NEXT:    movswl 4(%rdi), %eax
784; AVX512F-NEXT:    vmovd %eax, %xmm0
785; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm9
786; AVX512F-NEXT:    movswl (%rdi), %eax
787; AVX512F-NEXT:    vmovd %eax, %xmm0
788; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm10
789; AVX512F-NEXT:    movswl 2(%rdi), %eax
790; AVX512F-NEXT:    vmovd %eax, %xmm0
791; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm11
792; AVX512F-NEXT:    movswl 14(%rdi), %eax
793; AVX512F-NEXT:    vmovd %eax, %xmm0
794; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm12
795; AVX512F-NEXT:    movswl 12(%rdi), %eax
796; AVX512F-NEXT:    vmovd %eax, %xmm0
797; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm13
798; AVX512F-NEXT:    movswl 8(%rdi), %eax
799; AVX512F-NEXT:    vmovd %eax, %xmm0
800; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm14
801; AVX512F-NEXT:    movswl 10(%rdi), %eax
802; AVX512F-NEXT:    vmovd %eax, %xmm0
803; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm15
804; AVX512F-NEXT:    movswl 22(%rdi), %eax
805; AVX512F-NEXT:    vmovd %eax, %xmm0
806; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
807; AVX512F-NEXT:    movswl 20(%rdi), %eax
808; AVX512F-NEXT:    vmovd %eax, %xmm1
809; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
810; AVX512F-NEXT:    movswl 16(%rdi), %eax
811; AVX512F-NEXT:    vmovd %eax, %xmm2
812; AVX512F-NEXT:    vcvtph2ps %xmm2, %xmm2
813; AVX512F-NEXT:    movswl 18(%rdi), %eax
814; AVX512F-NEXT:    vmovd %eax, %xmm3
815; AVX512F-NEXT:    vcvtph2ps %xmm3, %xmm3
816; AVX512F-NEXT:    movswl 30(%rdi), %eax
817; AVX512F-NEXT:    vmovd %eax, %xmm4
818; AVX512F-NEXT:    vcvtph2ps %xmm4, %xmm4
819; AVX512F-NEXT:    movswl 28(%rdi), %eax
820; AVX512F-NEXT:    vmovd %eax, %xmm5
821; AVX512F-NEXT:    vcvtph2ps %xmm5, %xmm5
822; AVX512F-NEXT:    movswl 24(%rdi), %eax
823; AVX512F-NEXT:    vmovd %eax, %xmm6
824; AVX512F-NEXT:    vcvtph2ps %xmm6, %xmm6
825; AVX512F-NEXT:    movswl 26(%rdi), %eax
826; AVX512F-NEXT:    vmovd %eax, %xmm7
827; AVX512F-NEXT:    vcvtph2ps %xmm7, %xmm7
828; AVX512F-NEXT:    vinsertps {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[2,3]
829; AVX512F-NEXT:    vinsertps {{.*#+}} xmm5 = xmm6[0,1],xmm5[0],xmm6[3]
830; AVX512F-NEXT:    vinsertps {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[0]
831; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
832; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
833; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
834; AVX512F-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
835; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm14[0],xmm15[0],xmm14[2,3]
836; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm13[0],xmm1[3]
837; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm12[0]
838; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm10[0],xmm11[0],xmm10[2,3]
839; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
840; AVX512F-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
841; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
842; AVX512F-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
843; AVX512F-NEXT:    retq
844;
845; AVX512VL-LABEL: load_cvt_16i16_to_16f32:
846; AVX512VL:       # %bb.0:
847; AVX512VL-NEXT:    movswl 6(%rdi), %eax
848; AVX512VL-NEXT:    vmovd %eax, %xmm0
849; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm8
850; AVX512VL-NEXT:    movswl 4(%rdi), %eax
851; AVX512VL-NEXT:    vmovd %eax, %xmm1
852; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm9
853; AVX512VL-NEXT:    movswl (%rdi), %eax
854; AVX512VL-NEXT:    vmovd %eax, %xmm2
855; AVX512VL-NEXT:    vcvtph2ps %xmm2, %xmm10
856; AVX512VL-NEXT:    movswl 2(%rdi), %eax
857; AVX512VL-NEXT:    vmovd %eax, %xmm3
858; AVX512VL-NEXT:    vcvtph2ps %xmm3, %xmm11
859; AVX512VL-NEXT:    movswl 14(%rdi), %eax
860; AVX512VL-NEXT:    vmovd %eax, %xmm4
861; AVX512VL-NEXT:    vcvtph2ps %xmm4, %xmm12
862; AVX512VL-NEXT:    movswl 12(%rdi), %eax
863; AVX512VL-NEXT:    vmovd %eax, %xmm5
864; AVX512VL-NEXT:    vcvtph2ps %xmm5, %xmm13
865; AVX512VL-NEXT:    movswl 8(%rdi), %eax
866; AVX512VL-NEXT:    vmovd %eax, %xmm6
867; AVX512VL-NEXT:    vcvtph2ps %xmm6, %xmm14
868; AVX512VL-NEXT:    movswl 10(%rdi), %eax
869; AVX512VL-NEXT:    vmovd %eax, %xmm7
870; AVX512VL-NEXT:    vcvtph2ps %xmm7, %xmm15
871; AVX512VL-NEXT:    movswl 22(%rdi), %eax
872; AVX512VL-NEXT:    vmovd %eax, %xmm0
873; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
874; AVX512VL-NEXT:    movswl 20(%rdi), %eax
875; AVX512VL-NEXT:    vmovd %eax, %xmm1
876; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
877; AVX512VL-NEXT:    movswl 16(%rdi), %eax
878; AVX512VL-NEXT:    vmovd %eax, %xmm2
879; AVX512VL-NEXT:    vcvtph2ps %xmm2, %xmm2
880; AVX512VL-NEXT:    movswl 18(%rdi), %eax
881; AVX512VL-NEXT:    vmovd %eax, %xmm3
882; AVX512VL-NEXT:    vcvtph2ps %xmm3, %xmm3
883; AVX512VL-NEXT:    movswl 30(%rdi), %eax
884; AVX512VL-NEXT:    vmovd %eax, %xmm4
885; AVX512VL-NEXT:    vcvtph2ps %xmm4, %xmm4
886; AVX512VL-NEXT:    movswl 28(%rdi), %eax
887; AVX512VL-NEXT:    vmovd %eax, %xmm5
888; AVX512VL-NEXT:    vcvtph2ps %xmm5, %xmm5
889; AVX512VL-NEXT:    movswl 24(%rdi), %eax
890; AVX512VL-NEXT:    vmovd %eax, %xmm6
891; AVX512VL-NEXT:    vcvtph2ps %xmm6, %xmm6
892; AVX512VL-NEXT:    movswl 26(%rdi), %eax
893; AVX512VL-NEXT:    vmovd %eax, %xmm7
894; AVX512VL-NEXT:    vcvtph2ps %xmm7, %xmm7
895; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[2,3]
896; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm5 = xmm6[0,1],xmm5[0],xmm6[3]
897; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[0]
898; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
899; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1],xmm1[0],xmm2[3]
900; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
901; AVX512VL-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
902; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm14[0],xmm15[0],xmm14[2,3]
903; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm13[0],xmm1[3]
904; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm12[0]
905; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm10[0],xmm11[0],xmm10[2,3]
906; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm9[0],xmm2[3]
907; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[0]
908; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
909; AVX512VL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
910; AVX512VL-NEXT:    retq
911  %1 = load <16 x i16>, <16 x i16>* %a0
912  %2 = bitcast <16 x i16> %1 to <16 x half>
913  %3 = fpext <16 x half> %2 to <16 x float>
914  ret <16 x float> %3
915}
916
917;
918; Half to Double
919;
920
921define double @cvt_i16_to_f64(i16 %a0) nounwind {
922; ALL-LABEL: cvt_i16_to_f64:
923; ALL:       # %bb.0:
924; ALL-NEXT:    movswl %di, %eax
925; ALL-NEXT:    vmovd %eax, %xmm0
926; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
927; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
928; ALL-NEXT:    retq
929  %1 = bitcast i16 %a0 to half
930  %2 = fpext half %1 to double
931  ret double %2
932}
933
934define <2 x double> @cvt_2i16_to_2f64(<2 x i16> %a0) nounwind {
935; AVX1-LABEL: cvt_2i16_to_2f64:
936; AVX1:       # %bb.0:
937; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
938; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
939; AVX1-NEXT:    vmovd %xmm0, %eax
940; AVX1-NEXT:    movswl %ax, %ecx
941; AVX1-NEXT:    shrl $16, %eax
942; AVX1-NEXT:    cwtl
943; AVX1-NEXT:    vmovd %eax, %xmm0
944; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
945; AVX1-NEXT:    vmovd %ecx, %xmm1
946; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
947; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
948; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
949; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
950; AVX1-NEXT:    retq
951;
952; AVX2-SLOW-LABEL: cvt_2i16_to_2f64:
953; AVX2-SLOW:       # %bb.0:
954; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
955; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
956; AVX2-SLOW-NEXT:    vmovd %xmm0, %eax
957; AVX2-SLOW-NEXT:    movswl %ax, %ecx
958; AVX2-SLOW-NEXT:    shrl $16, %eax
959; AVX2-SLOW-NEXT:    cwtl
960; AVX2-SLOW-NEXT:    vmovd %eax, %xmm0
961; AVX2-SLOW-NEXT:    vcvtph2ps %xmm0, %xmm0
962; AVX2-SLOW-NEXT:    vmovd %ecx, %xmm1
963; AVX2-SLOW-NEXT:    vcvtph2ps %xmm1, %xmm1
964; AVX2-SLOW-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
965; AVX2-SLOW-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
966; AVX2-SLOW-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
967; AVX2-SLOW-NEXT:    retq
968;
969; AVX2-FAST-LABEL: cvt_2i16_to_2f64:
970; AVX2-FAST:       # %bb.0:
971; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,10,11,8,9,10,11,12,13,14,15]
972; AVX2-FAST-NEXT:    vmovd %xmm0, %eax
973; AVX2-FAST-NEXT:    movswl %ax, %ecx
974; AVX2-FAST-NEXT:    shrl $16, %eax
975; AVX2-FAST-NEXT:    cwtl
976; AVX2-FAST-NEXT:    vmovd %eax, %xmm0
977; AVX2-FAST-NEXT:    vcvtph2ps %xmm0, %xmm0
978; AVX2-FAST-NEXT:    vmovd %ecx, %xmm1
979; AVX2-FAST-NEXT:    vcvtph2ps %xmm1, %xmm1
980; AVX2-FAST-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
981; AVX2-FAST-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
982; AVX2-FAST-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
983; AVX2-FAST-NEXT:    retq
984;
985; AVX512F-LABEL: cvt_2i16_to_2f64:
986; AVX512F:       # %bb.0:
987; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
988; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
989; AVX512F-NEXT:    vmovd %xmm0, %eax
990; AVX512F-NEXT:    movswl %ax, %ecx
991; AVX512F-NEXT:    shrl $16, %eax
992; AVX512F-NEXT:    cwtl
993; AVX512F-NEXT:    vmovd %eax, %xmm0
994; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
995; AVX512F-NEXT:    vmovd %ecx, %xmm1
996; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
997; AVX512F-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
998; AVX512F-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
999; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1000; AVX512F-NEXT:    retq
1001;
1002; AVX512VL-LABEL: cvt_2i16_to_2f64:
1003; AVX512VL:       # %bb.0:
1004; AVX512VL-NEXT:    vpmovqw %xmm0, -{{[0-9]+}}(%rsp)
1005; AVX512VL-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
1006; AVX512VL-NEXT:    movswl %ax, %ecx
1007; AVX512VL-NEXT:    shrl $16, %eax
1008; AVX512VL-NEXT:    cwtl
1009; AVX512VL-NEXT:    vmovd %eax, %xmm0
1010; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
1011; AVX512VL-NEXT:    vmovd %ecx, %xmm1
1012; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
1013; AVX512VL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1014; AVX512VL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1015; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1016; AVX512VL-NEXT:    retq
1017  %1 = bitcast <2 x i16> %a0 to <2 x half>
1018  %2 = fpext <2 x half> %1 to <2 x double>
1019  ret <2 x double> %2
1020}
1021
1022define <4 x double> @cvt_4i16_to_4f64(<4 x i16> %a0) nounwind {
1023; ALL-LABEL: cvt_4i16_to_4f64:
1024; ALL:       # %bb.0:
1025; ALL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
1026; ALL-NEXT:    vmovq %xmm0, %rax
1027; ALL-NEXT:    movq %rax, %rcx
1028; ALL-NEXT:    movl %eax, %edx
1029; ALL-NEXT:    movswl %ax, %esi
1030; ALL-NEXT:    shrq $48, %rax
1031; ALL-NEXT:    shrq $32, %rcx
1032; ALL-NEXT:    shrl $16, %edx
1033; ALL-NEXT:    movswl %dx, %edx
1034; ALL-NEXT:    vmovd %edx, %xmm0
1035; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
1036; ALL-NEXT:    vmovd %esi, %xmm1
1037; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
1038; ALL-NEXT:    movswl %cx, %ecx
1039; ALL-NEXT:    vmovd %ecx, %xmm2
1040; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
1041; ALL-NEXT:    cwtl
1042; ALL-NEXT:    vmovd %eax, %xmm3
1043; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
1044; ALL-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1045; ALL-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1046; ALL-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1047; ALL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1048; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1049; ALL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1050; ALL-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1051; ALL-NEXT:    retq
1052  %1 = bitcast <4 x i16> %a0 to <4 x half>
1053  %2 = fpext <4 x half> %1 to <4 x double>
1054  ret <4 x double> %2
1055}
1056
1057define <2 x double> @cvt_8i16_to_2f64(<8 x i16> %a0) nounwind {
1058; AVX1-LABEL: cvt_8i16_to_2f64:
1059; AVX1:       # %bb.0:
1060; AVX1-NEXT:    vmovd %xmm0, %eax
1061; AVX1-NEXT:    movswl %ax, %ecx
1062; AVX1-NEXT:    shrl $16, %eax
1063; AVX1-NEXT:    cwtl
1064; AVX1-NEXT:    vmovd %eax, %xmm0
1065; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
1066; AVX1-NEXT:    vmovd %ecx, %xmm1
1067; AVX1-NEXT:    vcvtph2ps %xmm1, %xmm1
1068; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1069; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1070; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1071; AVX1-NEXT:    retq
1072;
1073; AVX2-LABEL: cvt_8i16_to_2f64:
1074; AVX2:       # %bb.0:
1075; AVX2-NEXT:    vmovd %xmm0, %eax
1076; AVX2-NEXT:    movswl %ax, %ecx
1077; AVX2-NEXT:    shrl $16, %eax
1078; AVX2-NEXT:    cwtl
1079; AVX2-NEXT:    vmovd %eax, %xmm0
1080; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
1081; AVX2-NEXT:    vmovd %ecx, %xmm1
1082; AVX2-NEXT:    vcvtph2ps %xmm1, %xmm1
1083; AVX2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1084; AVX2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1085; AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1086; AVX2-NEXT:    retq
1087;
1088; AVX512F-LABEL: cvt_8i16_to_2f64:
1089; AVX512F:       # %bb.0:
1090; AVX512F-NEXT:    vmovd %xmm0, %eax
1091; AVX512F-NEXT:    movswl %ax, %ecx
1092; AVX512F-NEXT:    shrl $16, %eax
1093; AVX512F-NEXT:    cwtl
1094; AVX512F-NEXT:    vmovd %eax, %xmm0
1095; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
1096; AVX512F-NEXT:    vmovd %ecx, %xmm1
1097; AVX512F-NEXT:    vcvtph2ps %xmm1, %xmm1
1098; AVX512F-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1099; AVX512F-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1100; AVX512F-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1101; AVX512F-NEXT:    retq
1102;
1103; AVX512VL-LABEL: cvt_8i16_to_2f64:
1104; AVX512VL:       # %bb.0:
1105; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
1106; AVX512VL-NEXT:    vpmovqw %xmm0, -{{[0-9]+}}(%rsp)
1107; AVX512VL-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
1108; AVX512VL-NEXT:    movswl %ax, %ecx
1109; AVX512VL-NEXT:    shrl $16, %eax
1110; AVX512VL-NEXT:    cwtl
1111; AVX512VL-NEXT:    vmovd %eax, %xmm0
1112; AVX512VL-NEXT:    vcvtph2ps %xmm0, %xmm0
1113; AVX512VL-NEXT:    vmovd %ecx, %xmm1
1114; AVX512VL-NEXT:    vcvtph2ps %xmm1, %xmm1
1115; AVX512VL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1116; AVX512VL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1117; AVX512VL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1118; AVX512VL-NEXT:    retq
1119  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
1120  %2 = bitcast <2 x i16> %1 to <2 x half>
1121  %3 = fpext <2 x half> %2 to <2 x double>
1122  ret <2 x double> %3
1123}
1124
1125define <4 x double> @cvt_8i16_to_4f64(<8 x i16> %a0) nounwind {
1126; ALL-LABEL: cvt_8i16_to_4f64:
1127; ALL:       # %bb.0:
1128; ALL-NEXT:    vmovq %xmm0, %rax
1129; ALL-NEXT:    movq %rax, %rcx
1130; ALL-NEXT:    movl %eax, %edx
1131; ALL-NEXT:    movswl %ax, %esi
1132; ALL-NEXT:    shrq $48, %rax
1133; ALL-NEXT:    shrq $32, %rcx
1134; ALL-NEXT:    shrl $16, %edx
1135; ALL-NEXT:    movswl %dx, %edx
1136; ALL-NEXT:    vmovd %edx, %xmm0
1137; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
1138; ALL-NEXT:    vmovd %esi, %xmm1
1139; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
1140; ALL-NEXT:    movswl %cx, %ecx
1141; ALL-NEXT:    vmovd %ecx, %xmm2
1142; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
1143; ALL-NEXT:    cwtl
1144; ALL-NEXT:    vmovd %eax, %xmm3
1145; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
1146; ALL-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1147; ALL-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1148; ALL-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1149; ALL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1150; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1151; ALL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1152; ALL-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1153; ALL-NEXT:    retq
1154  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1155  %2 = bitcast <4 x i16> %1 to <4 x half>
1156  %3 = fpext <4 x half> %2 to <4 x double>
1157  ret <4 x double> %3
1158}
1159
1160define <8 x double> @cvt_8i16_to_8f64(<8 x i16> %a0) nounwind {
1161; AVX1-LABEL: cvt_8i16_to_8f64:
1162; AVX1:       # %bb.0:
1163; AVX1-NEXT:    vmovq %xmm0, %rdx
1164; AVX1-NEXT:    movq %rdx, %r9
1165; AVX1-NEXT:    movl %edx, %r10d
1166; AVX1-NEXT:    movswl %dx, %r8d
1167; AVX1-NEXT:    shrq $48, %rdx
1168; AVX1-NEXT:    shrq $32, %r9
1169; AVX1-NEXT:    shrl $16, %r10d
1170; AVX1-NEXT:    vpextrq $1, %xmm0, %rdi
1171; AVX1-NEXT:    movq %rdi, %rsi
1172; AVX1-NEXT:    movl %edi, %eax
1173; AVX1-NEXT:    movswl %di, %ecx
1174; AVX1-NEXT:    shrq $48, %rdi
1175; AVX1-NEXT:    shrq $32, %rsi
1176; AVX1-NEXT:    shrl $16, %eax
1177; AVX1-NEXT:    cwtl
1178; AVX1-NEXT:    vmovd %eax, %xmm0
1179; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm1
1180; AVX1-NEXT:    vmovd %ecx, %xmm0
1181; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm2
1182; AVX1-NEXT:    movswl %si, %eax
1183; AVX1-NEXT:    vmovd %eax, %xmm0
1184; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm3
1185; AVX1-NEXT:    movswl %di, %eax
1186; AVX1-NEXT:    vmovd %eax, %xmm0
1187; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm4
1188; AVX1-NEXT:    movswl %r10w, %eax
1189; AVX1-NEXT:    vmovd %eax, %xmm0
1190; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
1191; AVX1-NEXT:    vmovd %r8d, %xmm5
1192; AVX1-NEXT:    vcvtph2ps %xmm5, %xmm5
1193; AVX1-NEXT:    movswl %r9w, %eax
1194; AVX1-NEXT:    vmovd %eax, %xmm6
1195; AVX1-NEXT:    vcvtph2ps %xmm6, %xmm6
1196; AVX1-NEXT:    movswl %dx, %eax
1197; AVX1-NEXT:    vmovd %eax, %xmm7
1198; AVX1-NEXT:    vcvtph2ps %xmm7, %xmm7
1199; AVX1-NEXT:    vcvtss2sd %xmm7, %xmm7, %xmm7
1200; AVX1-NEXT:    vcvtss2sd %xmm6, %xmm6, %xmm6
1201; AVX1-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
1202; AVX1-NEXT:    vcvtss2sd %xmm5, %xmm5, %xmm5
1203; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1204; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm5[0],xmm0[0]
1205; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
1206; AVX1-NEXT:    vcvtss2sd %xmm4, %xmm4, %xmm4
1207; AVX1-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1208; AVX1-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
1209; AVX1-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1210; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1211; AVX1-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0]
1212; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
1213; AVX1-NEXT:    retq
1214;
1215; AVX2-LABEL: cvt_8i16_to_8f64:
1216; AVX2:       # %bb.0:
1217; AVX2-NEXT:    vmovq %xmm0, %rdx
1218; AVX2-NEXT:    movq %rdx, %r9
1219; AVX2-NEXT:    movl %edx, %r10d
1220; AVX2-NEXT:    movswl %dx, %r8d
1221; AVX2-NEXT:    shrq $48, %rdx
1222; AVX2-NEXT:    shrq $32, %r9
1223; AVX2-NEXT:    shrl $16, %r10d
1224; AVX2-NEXT:    vpextrq $1, %xmm0, %rdi
1225; AVX2-NEXT:    movq %rdi, %rsi
1226; AVX2-NEXT:    movl %edi, %eax
1227; AVX2-NEXT:    movswl %di, %ecx
1228; AVX2-NEXT:    shrq $48, %rdi
1229; AVX2-NEXT:    shrq $32, %rsi
1230; AVX2-NEXT:    shrl $16, %eax
1231; AVX2-NEXT:    cwtl
1232; AVX2-NEXT:    vmovd %eax, %xmm0
1233; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm1
1234; AVX2-NEXT:    vmovd %ecx, %xmm0
1235; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm2
1236; AVX2-NEXT:    movswl %si, %eax
1237; AVX2-NEXT:    vmovd %eax, %xmm0
1238; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm3
1239; AVX2-NEXT:    movswl %di, %eax
1240; AVX2-NEXT:    vmovd %eax, %xmm0
1241; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm4
1242; AVX2-NEXT:    movswl %r10w, %eax
1243; AVX2-NEXT:    vmovd %eax, %xmm0
1244; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
1245; AVX2-NEXT:    vmovd %r8d, %xmm5
1246; AVX2-NEXT:    vcvtph2ps %xmm5, %xmm5
1247; AVX2-NEXT:    movswl %r9w, %eax
1248; AVX2-NEXT:    vmovd %eax, %xmm6
1249; AVX2-NEXT:    vcvtph2ps %xmm6, %xmm6
1250; AVX2-NEXT:    movswl %dx, %eax
1251; AVX2-NEXT:    vmovd %eax, %xmm7
1252; AVX2-NEXT:    vcvtph2ps %xmm7, %xmm7
1253; AVX2-NEXT:    vcvtss2sd %xmm7, %xmm7, %xmm7
1254; AVX2-NEXT:    vcvtss2sd %xmm6, %xmm6, %xmm6
1255; AVX2-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
1256; AVX2-NEXT:    vcvtss2sd %xmm5, %xmm5, %xmm5
1257; AVX2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1258; AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm5[0],xmm0[0]
1259; AVX2-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
1260; AVX2-NEXT:    vcvtss2sd %xmm4, %xmm4, %xmm4
1261; AVX2-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1262; AVX2-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
1263; AVX2-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1264; AVX2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1265; AVX2-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0]
1266; AVX2-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
1267; AVX2-NEXT:    retq
1268;
1269; AVX512-LABEL: cvt_8i16_to_8f64:
1270; AVX512:       # %bb.0:
1271; AVX512-NEXT:    vpextrq $1, %xmm0, %rdx
1272; AVX512-NEXT:    movq %rdx, %r9
1273; AVX512-NEXT:    movl %edx, %r10d
1274; AVX512-NEXT:    movswl %dx, %r8d
1275; AVX512-NEXT:    shrq $48, %rdx
1276; AVX512-NEXT:    shrq $32, %r9
1277; AVX512-NEXT:    shrl $16, %r10d
1278; AVX512-NEXT:    vmovq %xmm0, %rdi
1279; AVX512-NEXT:    movq %rdi, %rsi
1280; AVX512-NEXT:    movl %edi, %eax
1281; AVX512-NEXT:    movswl %di, %ecx
1282; AVX512-NEXT:    shrq $48, %rdi
1283; AVX512-NEXT:    shrq $32, %rsi
1284; AVX512-NEXT:    shrl $16, %eax
1285; AVX512-NEXT:    cwtl
1286; AVX512-NEXT:    vmovd %eax, %xmm0
1287; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
1288; AVX512-NEXT:    vmovd %ecx, %xmm1
1289; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
1290; AVX512-NEXT:    movswl %si, %eax
1291; AVX512-NEXT:    vmovd %eax, %xmm2
1292; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
1293; AVX512-NEXT:    movswl %di, %eax
1294; AVX512-NEXT:    vmovd %eax, %xmm3
1295; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
1296; AVX512-NEXT:    movswl %r10w, %eax
1297; AVX512-NEXT:    vmovd %eax, %xmm4
1298; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm4
1299; AVX512-NEXT:    vmovd %r8d, %xmm5
1300; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
1301; AVX512-NEXT:    movswl %r9w, %eax
1302; AVX512-NEXT:    vmovd %eax, %xmm6
1303; AVX512-NEXT:    vcvtph2ps %xmm6, %xmm6
1304; AVX512-NEXT:    movswl %dx, %eax
1305; AVX512-NEXT:    vmovd %eax, %xmm7
1306; AVX512-NEXT:    vcvtph2ps %xmm7, %xmm7
1307; AVX512-NEXT:    vcvtss2sd %xmm7, %xmm7, %xmm7
1308; AVX512-NEXT:    vcvtss2sd %xmm6, %xmm6, %xmm6
1309; AVX512-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
1310; AVX512-NEXT:    vcvtss2sd %xmm5, %xmm5, %xmm5
1311; AVX512-NEXT:    vcvtss2sd %xmm4, %xmm4, %xmm4
1312; AVX512-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm5[0],xmm4[0]
1313; AVX512-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm4
1314; AVX512-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1315; AVX512-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1316; AVX512-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1317; AVX512-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1318; AVX512-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1319; AVX512-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1320; AVX512-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1321; AVX512-NEXT:    vinsertf64x4 $1, %ymm4, %zmm0, %zmm0
1322; AVX512-NEXT:    retq
1323  %1 = bitcast <8 x i16> %a0 to <8 x half>
1324  %2 = fpext <8 x half> %1 to <8 x double>
1325  ret <8 x double> %2
1326}
1327
1328;
1329; Half to Double (Load)
1330;
1331
1332define double @load_cvt_i16_to_f64(i16* %a0) nounwind {
1333; ALL-LABEL: load_cvt_i16_to_f64:
1334; ALL:       # %bb.0:
1335; ALL-NEXT:    movswl (%rdi), %eax
1336; ALL-NEXT:    vmovd %eax, %xmm0
1337; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
1338; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1339; ALL-NEXT:    retq
1340  %1 = load i16, i16* %a0
1341  %2 = bitcast i16 %1 to half
1342  %3 = fpext half %2 to double
1343  ret double %3
1344}
1345
1346define <2 x double> @load_cvt_2i16_to_2f64(<2 x i16>* %a0) nounwind {
1347; ALL-LABEL: load_cvt_2i16_to_2f64:
1348; ALL:       # %bb.0:
1349; ALL-NEXT:    movswl (%rdi), %eax
1350; ALL-NEXT:    vmovd %eax, %xmm0
1351; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
1352; ALL-NEXT:    movswl 2(%rdi), %eax
1353; ALL-NEXT:    vmovd %eax, %xmm1
1354; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
1355; ALL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1356; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1357; ALL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1358; ALL-NEXT:    retq
1359  %1 = load <2 x i16>, <2 x i16>* %a0
1360  %2 = bitcast <2 x i16> %1 to <2 x half>
1361  %3 = fpext <2 x half> %2 to <2 x double>
1362  ret <2 x double> %3
1363}
1364
1365define <4 x double> @load_cvt_4i16_to_4f64(<4 x i16>* %a0) nounwind {
1366; ALL-LABEL: load_cvt_4i16_to_4f64:
1367; ALL:       # %bb.0:
1368; ALL-NEXT:    movswl (%rdi), %eax
1369; ALL-NEXT:    vmovd %eax, %xmm0
1370; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
1371; ALL-NEXT:    movswl 2(%rdi), %eax
1372; ALL-NEXT:    vmovd %eax, %xmm1
1373; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
1374; ALL-NEXT:    movswl 4(%rdi), %eax
1375; ALL-NEXT:    vmovd %eax, %xmm2
1376; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
1377; ALL-NEXT:    movswl 6(%rdi), %eax
1378; ALL-NEXT:    vmovd %eax, %xmm3
1379; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
1380; ALL-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1381; ALL-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1382; ALL-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1383; ALL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1384; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1385; ALL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1386; ALL-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1387; ALL-NEXT:    retq
1388  %1 = load <4 x i16>, <4 x i16>* %a0
1389  %2 = bitcast <4 x i16> %1 to <4 x half>
1390  %3 = fpext <4 x half> %2 to <4 x double>
1391  ret <4 x double> %3
1392}
1393
1394define <4 x double> @load_cvt_8i16_to_4f64(<8 x i16>* %a0) nounwind {
1395; ALL-LABEL: load_cvt_8i16_to_4f64:
1396; ALL:       # %bb.0:
1397; ALL-NEXT:    movq (%rdi), %rax
1398; ALL-NEXT:    movq %rax, %rcx
1399; ALL-NEXT:    movl %eax, %edx
1400; ALL-NEXT:    movswl %ax, %esi
1401; ALL-NEXT:    shrq $48, %rax
1402; ALL-NEXT:    shrq $32, %rcx
1403; ALL-NEXT:    shrl $16, %edx
1404; ALL-NEXT:    movswl %dx, %edx
1405; ALL-NEXT:    vmovd %edx, %xmm0
1406; ALL-NEXT:    vcvtph2ps %xmm0, %xmm0
1407; ALL-NEXT:    vmovd %esi, %xmm1
1408; ALL-NEXT:    vcvtph2ps %xmm1, %xmm1
1409; ALL-NEXT:    movswl %cx, %ecx
1410; ALL-NEXT:    vmovd %ecx, %xmm2
1411; ALL-NEXT:    vcvtph2ps %xmm2, %xmm2
1412; ALL-NEXT:    cwtl
1413; ALL-NEXT:    vmovd %eax, %xmm3
1414; ALL-NEXT:    vcvtph2ps %xmm3, %xmm3
1415; ALL-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1416; ALL-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1417; ALL-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1418; ALL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1419; ALL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1420; ALL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1421; ALL-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1422; ALL-NEXT:    retq
1423  %1 = load <8 x i16>, <8 x i16>* %a0
1424  %2 = shufflevector <8 x i16> %1, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1425  %3 = bitcast <4 x i16> %2 to <4 x half>
1426  %4 = fpext <4 x half> %3 to <4 x double>
1427  ret <4 x double> %4
1428}
1429
1430define <8 x double> @load_cvt_8i16_to_8f64(<8 x i16>* %a0) nounwind {
1431; AVX1-LABEL: load_cvt_8i16_to_8f64:
1432; AVX1:       # %bb.0:
1433; AVX1-NEXT:    movswl 8(%rdi), %eax
1434; AVX1-NEXT:    vmovd %eax, %xmm0
1435; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm1
1436; AVX1-NEXT:    movswl 10(%rdi), %eax
1437; AVX1-NEXT:    vmovd %eax, %xmm0
1438; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm2
1439; AVX1-NEXT:    movswl 12(%rdi), %eax
1440; AVX1-NEXT:    vmovd %eax, %xmm0
1441; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm3
1442; AVX1-NEXT:    movswl 14(%rdi), %eax
1443; AVX1-NEXT:    vmovd %eax, %xmm0
1444; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm4
1445; AVX1-NEXT:    movswl (%rdi), %eax
1446; AVX1-NEXT:    vmovd %eax, %xmm0
1447; AVX1-NEXT:    vcvtph2ps %xmm0, %xmm0
1448; AVX1-NEXT:    movswl 2(%rdi), %eax
1449; AVX1-NEXT:    vmovd %eax, %xmm5
1450; AVX1-NEXT:    vcvtph2ps %xmm5, %xmm5
1451; AVX1-NEXT:    movswl 4(%rdi), %eax
1452; AVX1-NEXT:    vmovd %eax, %xmm6
1453; AVX1-NEXT:    vcvtph2ps %xmm6, %xmm6
1454; AVX1-NEXT:    movswl 6(%rdi), %eax
1455; AVX1-NEXT:    vmovd %eax, %xmm7
1456; AVX1-NEXT:    vcvtph2ps %xmm7, %xmm7
1457; AVX1-NEXT:    vcvtss2sd %xmm7, %xmm7, %xmm7
1458; AVX1-NEXT:    vcvtss2sd %xmm6, %xmm6, %xmm6
1459; AVX1-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
1460; AVX1-NEXT:    vcvtss2sd %xmm5, %xmm5, %xmm5
1461; AVX1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1462; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm5[0]
1463; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
1464; AVX1-NEXT:    vcvtss2sd %xmm4, %xmm4, %xmm4
1465; AVX1-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1466; AVX1-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
1467; AVX1-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1468; AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1469; AVX1-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
1470; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
1471; AVX1-NEXT:    retq
1472;
1473; AVX2-LABEL: load_cvt_8i16_to_8f64:
1474; AVX2:       # %bb.0:
1475; AVX2-NEXT:    movswl 8(%rdi), %eax
1476; AVX2-NEXT:    vmovd %eax, %xmm0
1477; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm1
1478; AVX2-NEXT:    movswl 10(%rdi), %eax
1479; AVX2-NEXT:    vmovd %eax, %xmm0
1480; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm2
1481; AVX2-NEXT:    movswl 12(%rdi), %eax
1482; AVX2-NEXT:    vmovd %eax, %xmm0
1483; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm3
1484; AVX2-NEXT:    movswl 14(%rdi), %eax
1485; AVX2-NEXT:    vmovd %eax, %xmm0
1486; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm4
1487; AVX2-NEXT:    movswl (%rdi), %eax
1488; AVX2-NEXT:    vmovd %eax, %xmm0
1489; AVX2-NEXT:    vcvtph2ps %xmm0, %xmm0
1490; AVX2-NEXT:    movswl 2(%rdi), %eax
1491; AVX2-NEXT:    vmovd %eax, %xmm5
1492; AVX2-NEXT:    vcvtph2ps %xmm5, %xmm5
1493; AVX2-NEXT:    movswl 4(%rdi), %eax
1494; AVX2-NEXT:    vmovd %eax, %xmm6
1495; AVX2-NEXT:    vcvtph2ps %xmm6, %xmm6
1496; AVX2-NEXT:    movswl 6(%rdi), %eax
1497; AVX2-NEXT:    vmovd %eax, %xmm7
1498; AVX2-NEXT:    vcvtph2ps %xmm7, %xmm7
1499; AVX2-NEXT:    vcvtss2sd %xmm7, %xmm7, %xmm7
1500; AVX2-NEXT:    vcvtss2sd %xmm6, %xmm6, %xmm6
1501; AVX2-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
1502; AVX2-NEXT:    vcvtss2sd %xmm5, %xmm5, %xmm5
1503; AVX2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1504; AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm5[0]
1505; AVX2-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
1506; AVX2-NEXT:    vcvtss2sd %xmm4, %xmm4, %xmm4
1507; AVX2-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1508; AVX2-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
1509; AVX2-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1510; AVX2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1511; AVX2-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
1512; AVX2-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
1513; AVX2-NEXT:    retq
1514;
1515; AVX512-LABEL: load_cvt_8i16_to_8f64:
1516; AVX512:       # %bb.0:
1517; AVX512-NEXT:    movswl (%rdi), %eax
1518; AVX512-NEXT:    vmovd %eax, %xmm0
1519; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
1520; AVX512-NEXT:    movswl 2(%rdi), %eax
1521; AVX512-NEXT:    vmovd %eax, %xmm1
1522; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
1523; AVX512-NEXT:    movswl 4(%rdi), %eax
1524; AVX512-NEXT:    vmovd %eax, %xmm2
1525; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
1526; AVX512-NEXT:    movswl 6(%rdi), %eax
1527; AVX512-NEXT:    vmovd %eax, %xmm3
1528; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
1529; AVX512-NEXT:    movswl 8(%rdi), %eax
1530; AVX512-NEXT:    vmovd %eax, %xmm4
1531; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm4
1532; AVX512-NEXT:    movswl 10(%rdi), %eax
1533; AVX512-NEXT:    vmovd %eax, %xmm5
1534; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
1535; AVX512-NEXT:    movswl 12(%rdi), %eax
1536; AVX512-NEXT:    vmovd %eax, %xmm6
1537; AVX512-NEXT:    vcvtph2ps %xmm6, %xmm6
1538; AVX512-NEXT:    movswl 14(%rdi), %eax
1539; AVX512-NEXT:    vmovd %eax, %xmm7
1540; AVX512-NEXT:    vcvtph2ps %xmm7, %xmm7
1541; AVX512-NEXT:    vcvtss2sd %xmm7, %xmm7, %xmm7
1542; AVX512-NEXT:    vcvtss2sd %xmm6, %xmm6, %xmm6
1543; AVX512-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm7[0]
1544; AVX512-NEXT:    vcvtss2sd %xmm5, %xmm5, %xmm5
1545; AVX512-NEXT:    vcvtss2sd %xmm4, %xmm4, %xmm4
1546; AVX512-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm4[0],xmm5[0]
1547; AVX512-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm4
1548; AVX512-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
1549; AVX512-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
1550; AVX512-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
1551; AVX512-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
1552; AVX512-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
1553; AVX512-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1554; AVX512-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1555; AVX512-NEXT:    vinsertf64x4 $1, %ymm4, %zmm0, %zmm0
1556; AVX512-NEXT:    retq
1557  %1 = load <8 x i16>, <8 x i16>* %a0
1558  %2 = bitcast <8 x i16> %1 to <8 x half>
1559  %3 = fpext <8 x half> %2 to <8 x double>
1560  ret <8 x double> %3
1561}
1562
1563;
1564; Float to Half
1565;
1566
1567define i16 @cvt_f32_to_i16(float %a0) nounwind {
1568; ALL-LABEL: cvt_f32_to_i16:
1569; ALL:       # %bb.0:
1570; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1571; ALL-NEXT:    vmovd %xmm0, %eax
1572; ALL-NEXT:    # kill: def $ax killed $ax killed $eax
1573; ALL-NEXT:    retq
1574  %1 = fptrunc float %a0 to half
1575  %2 = bitcast half %1 to i16
1576  ret i16 %2
1577}
1578
1579define <4 x i16> @cvt_4f32_to_4i16(<4 x float> %a0) nounwind {
1580; ALL-LABEL: cvt_4f32_to_4i16:
1581; ALL:       # %bb.0:
1582; ALL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
1583; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1584; ALL-NEXT:    vmovd %xmm1, %eax
1585; ALL-NEXT:    shll $16, %eax
1586; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
1587; ALL-NEXT:    vmovd %xmm1, %ecx
1588; ALL-NEXT:    movzwl %cx, %ecx
1589; ALL-NEXT:    orl %eax, %ecx
1590; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
1591; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1592; ALL-NEXT:    vmovd %xmm1, %eax
1593; ALL-NEXT:    shll $16, %eax
1594; ALL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
1595; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1596; ALL-NEXT:    vmovd %xmm0, %edx
1597; ALL-NEXT:    movzwl %dx, %edx
1598; ALL-NEXT:    orl %eax, %edx
1599; ALL-NEXT:    shlq $32, %rdx
1600; ALL-NEXT:    orq %rcx, %rdx
1601; ALL-NEXT:    vmovq %rdx, %xmm0
1602; ALL-NEXT:    retq
1603  %1 = fptrunc <4 x float> %a0 to <4 x half>
1604  %2 = bitcast <4 x half> %1 to <4 x i16>
1605  ret <4 x i16> %2
1606}
1607
1608define <8 x i16> @cvt_4f32_to_8i16_undef(<4 x float> %a0) nounwind {
1609; ALL-LABEL: cvt_4f32_to_8i16_undef:
1610; ALL:       # %bb.0:
1611; ALL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
1612; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1613; ALL-NEXT:    vmovd %xmm1, %eax
1614; ALL-NEXT:    shll $16, %eax
1615; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
1616; ALL-NEXT:    vmovd %xmm1, %ecx
1617; ALL-NEXT:    movzwl %cx, %ecx
1618; ALL-NEXT:    orl %eax, %ecx
1619; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
1620; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1621; ALL-NEXT:    vmovd %xmm1, %eax
1622; ALL-NEXT:    shll $16, %eax
1623; ALL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
1624; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1625; ALL-NEXT:    vmovd %xmm0, %edx
1626; ALL-NEXT:    movzwl %dx, %edx
1627; ALL-NEXT:    orl %eax, %edx
1628; ALL-NEXT:    shlq $32, %rdx
1629; ALL-NEXT:    orq %rcx, %rdx
1630; ALL-NEXT:    vmovq %rdx, %xmm0
1631; ALL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
1632; ALL-NEXT:    retq
1633  %1 = fptrunc <4 x float> %a0 to <4 x half>
1634  %2 = bitcast <4 x half> %1 to <4 x i16>
1635  %3 = shufflevector <4 x i16> %2, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
1636  ret <8 x i16> %3
1637}
1638
1639define <8 x i16> @cvt_4f32_to_8i16_zero(<4 x float> %a0) nounwind {
1640; AVX1-LABEL: cvt_4f32_to_8i16_zero:
1641; AVX1:       # %bb.0:
1642; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
1643; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1644; AVX1-NEXT:    vmovd %xmm1, %eax
1645; AVX1-NEXT:    shll $16, %eax
1646; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
1647; AVX1-NEXT:    vmovd %xmm1, %ecx
1648; AVX1-NEXT:    movzwl %cx, %ecx
1649; AVX1-NEXT:    orl %eax, %ecx
1650; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
1651; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1652; AVX1-NEXT:    vmovd %xmm1, %eax
1653; AVX1-NEXT:    shll $16, %eax
1654; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
1655; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1656; AVX1-NEXT:    vmovd %xmm0, %edx
1657; AVX1-NEXT:    movzwl %dx, %edx
1658; AVX1-NEXT:    orl %eax, %edx
1659; AVX1-NEXT:    shlq $32, %rdx
1660; AVX1-NEXT:    orq %rcx, %rdx
1661; AVX1-NEXT:    vmovq %rdx, %xmm0
1662; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
1663; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
1664; AVX1-NEXT:    retq
1665;
1666; AVX2-SLOW-LABEL: cvt_4f32_to_8i16_zero:
1667; AVX2-SLOW:       # %bb.0:
1668; AVX2-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
1669; AVX2-SLOW-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1670; AVX2-SLOW-NEXT:    vmovd %xmm1, %eax
1671; AVX2-SLOW-NEXT:    shll $16, %eax
1672; AVX2-SLOW-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
1673; AVX2-SLOW-NEXT:    vmovd %xmm1, %ecx
1674; AVX2-SLOW-NEXT:    movzwl %cx, %ecx
1675; AVX2-SLOW-NEXT:    orl %eax, %ecx
1676; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
1677; AVX2-SLOW-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1678; AVX2-SLOW-NEXT:    vmovd %xmm1, %eax
1679; AVX2-SLOW-NEXT:    shll $16, %eax
1680; AVX2-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
1681; AVX2-SLOW-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1682; AVX2-SLOW-NEXT:    vmovd %xmm0, %edx
1683; AVX2-SLOW-NEXT:    movzwl %dx, %edx
1684; AVX2-SLOW-NEXT:    orl %eax, %edx
1685; AVX2-SLOW-NEXT:    shlq $32, %rdx
1686; AVX2-SLOW-NEXT:    orq %rcx, %rdx
1687; AVX2-SLOW-NEXT:    vmovq %rdx, %xmm0
1688; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
1689; AVX2-SLOW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
1690; AVX2-SLOW-NEXT:    retq
1691;
1692; AVX2-FAST-LABEL: cvt_4f32_to_8i16_zero:
1693; AVX2-FAST:       # %bb.0:
1694; AVX2-FAST-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
1695; AVX2-FAST-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1696; AVX2-FAST-NEXT:    vmovd %xmm1, %eax
1697; AVX2-FAST-NEXT:    shll $16, %eax
1698; AVX2-FAST-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
1699; AVX2-FAST-NEXT:    vmovd %xmm1, %ecx
1700; AVX2-FAST-NEXT:    movzwl %cx, %ecx
1701; AVX2-FAST-NEXT:    orl %eax, %ecx
1702; AVX2-FAST-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
1703; AVX2-FAST-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1704; AVX2-FAST-NEXT:    vmovd %xmm1, %eax
1705; AVX2-FAST-NEXT:    shll $16, %eax
1706; AVX2-FAST-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
1707; AVX2-FAST-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1708; AVX2-FAST-NEXT:    vmovd %xmm0, %edx
1709; AVX2-FAST-NEXT:    movzwl %dx, %edx
1710; AVX2-FAST-NEXT:    orl %eax, %edx
1711; AVX2-FAST-NEXT:    shlq $32, %rdx
1712; AVX2-FAST-NEXT:    orq %rcx, %rdx
1713; AVX2-FAST-NEXT:    vmovq %rdx, %xmm0
1714; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero
1715; AVX2-FAST-NEXT:    retq
1716;
1717; AVX512F-LABEL: cvt_4f32_to_8i16_zero:
1718; AVX512F:       # %bb.0:
1719; AVX512F-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
1720; AVX512F-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1721; AVX512F-NEXT:    vmovd %xmm1, %eax
1722; AVX512F-NEXT:    shll $16, %eax
1723; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
1724; AVX512F-NEXT:    vmovd %xmm1, %ecx
1725; AVX512F-NEXT:    movzwl %cx, %ecx
1726; AVX512F-NEXT:    orl %eax, %ecx
1727; AVX512F-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
1728; AVX512F-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1729; AVX512F-NEXT:    vmovd %xmm1, %eax
1730; AVX512F-NEXT:    shll $16, %eax
1731; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
1732; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1733; AVX512F-NEXT:    vmovd %xmm0, %edx
1734; AVX512F-NEXT:    movzwl %dx, %edx
1735; AVX512F-NEXT:    orl %eax, %edx
1736; AVX512F-NEXT:    shlq $32, %rdx
1737; AVX512F-NEXT:    orq %rcx, %rdx
1738; AVX512F-NEXT:    vmovq %rdx, %xmm0
1739; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
1740; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
1741; AVX512F-NEXT:    retq
1742;
1743; AVX512VL-LABEL: cvt_4f32_to_8i16_zero:
1744; AVX512VL:       # %bb.0:
1745; AVX512VL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
1746; AVX512VL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1747; AVX512VL-NEXT:    vmovd %xmm1, %eax
1748; AVX512VL-NEXT:    shll $16, %eax
1749; AVX512VL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
1750; AVX512VL-NEXT:    vmovd %xmm1, %ecx
1751; AVX512VL-NEXT:    movzwl %cx, %ecx
1752; AVX512VL-NEXT:    orl %eax, %ecx
1753; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
1754; AVX512VL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1755; AVX512VL-NEXT:    vmovd %xmm1, %eax
1756; AVX512VL-NEXT:    shll $16, %eax
1757; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
1758; AVX512VL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1759; AVX512VL-NEXT:    vmovd %xmm0, %edx
1760; AVX512VL-NEXT:    movzwl %dx, %edx
1761; AVX512VL-NEXT:    orl %eax, %edx
1762; AVX512VL-NEXT:    shlq $32, %rdx
1763; AVX512VL-NEXT:    orq %rcx, %rdx
1764; AVX512VL-NEXT:    vmovq %rdx, %xmm0
1765; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero
1766; AVX512VL-NEXT:    retq
1767  %1 = fptrunc <4 x float> %a0 to <4 x half>
1768  %2 = bitcast <4 x half> %1 to <4 x i16>
1769  %3 = shufflevector <4 x i16> %2, <4 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
1770  ret <8 x i16> %3
1771}
1772
1773define <8 x i16> @cvt_8f32_to_8i16(<8 x float> %a0) nounwind {
1774; ALL-LABEL: cvt_8f32_to_8i16:
1775; ALL:       # %bb.0:
1776; ALL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
1777; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1778; ALL-NEXT:    vmovd %xmm1, %eax
1779; ALL-NEXT:    shll $16, %eax
1780; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
1781; ALL-NEXT:    vmovd %xmm1, %ecx
1782; ALL-NEXT:    movzwl %cx, %ecx
1783; ALL-NEXT:    orl %eax, %ecx
1784; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
1785; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1786; ALL-NEXT:    vmovd %xmm1, %edx
1787; ALL-NEXT:    shll $16, %edx
1788; ALL-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
1789; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1790; ALL-NEXT:    vmovd %xmm1, %eax
1791; ALL-NEXT:    movzwl %ax, %eax
1792; ALL-NEXT:    orl %edx, %eax
1793; ALL-NEXT:    shlq $32, %rax
1794; ALL-NEXT:    orq %rcx, %rax
1795; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
1796; ALL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
1797; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1798; ALL-NEXT:    vmovd %xmm1, %ecx
1799; ALL-NEXT:    shll $16, %ecx
1800; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
1801; ALL-NEXT:    vmovd %xmm1, %edx
1802; ALL-NEXT:    movzwl %dx, %edx
1803; ALL-NEXT:    orl %ecx, %edx
1804; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
1805; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1806; ALL-NEXT:    vmovd %xmm1, %ecx
1807; ALL-NEXT:    shll $16, %ecx
1808; ALL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
1809; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1810; ALL-NEXT:    vmovd %xmm0, %esi
1811; ALL-NEXT:    movzwl %si, %esi
1812; ALL-NEXT:    orl %ecx, %esi
1813; ALL-NEXT:    shlq $32, %rsi
1814; ALL-NEXT:    orq %rdx, %rsi
1815; ALL-NEXT:    vmovq %rsi, %xmm0
1816; ALL-NEXT:    vmovq %rax, %xmm1
1817; ALL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1818; ALL-NEXT:    vzeroupper
1819; ALL-NEXT:    retq
1820  %1 = fptrunc <8 x float> %a0 to <8 x half>
1821  %2 = bitcast <8 x half> %1 to <8 x i16>
1822  ret <8 x i16> %2
1823}
1824
1825define <16 x i16> @cvt_16f32_to_16i16(<16 x float> %a0) nounwind {
1826; AVX1-LABEL: cvt_16f32_to_16i16:
1827; AVX1:       # %bb.0:
1828; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm2
1829; AVX1-NEXT:    vmovd %xmm2, %eax
1830; AVX1-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
1831; AVX1-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
1832; AVX1-NEXT:    vmovd %eax, %xmm3
1833; AVX1-NEXT:    vmovd %xmm2, %eax
1834; AVX1-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
1835; AVX1-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
1836; AVX1-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
1837; AVX1-NEXT:    vmovd %xmm2, %eax
1838; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
1839; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
1840; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1841; AVX1-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
1842; AVX1-NEXT:    vmovd %xmm1, %eax
1843; AVX1-NEXT:    vcvtps2ph $4, %xmm2, %xmm1
1844; AVX1-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
1845; AVX1-NEXT:    vmovd %xmm1, %eax
1846; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
1847; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1848; AVX1-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
1849; AVX1-NEXT:    vmovd %xmm1, %eax
1850; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm2[1,0]
1851; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1852; AVX1-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
1853; AVX1-NEXT:    vmovd %xmm1, %eax
1854; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
1855; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
1856; AVX1-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
1857; AVX1-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm3
1858; AVX1-NEXT:    vmovd %xmm2, %eax
1859; AVX1-NEXT:    vpinsrw $7, %eax, %xmm3, %xmm2
1860; AVX1-NEXT:    vmovd %xmm1, %eax
1861; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
1862; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1863; AVX1-NEXT:    vmovd %eax, %xmm3
1864; AVX1-NEXT:    vmovd %xmm1, %eax
1865; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
1866; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1867; AVX1-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
1868; AVX1-NEXT:    vmovd %xmm1, %eax
1869; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1870; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
1871; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1872; AVX1-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
1873; AVX1-NEXT:    vmovd %xmm0, %eax
1874; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm0
1875; AVX1-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
1876; AVX1-NEXT:    vmovd %xmm0, %eax
1877; AVX1-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
1878; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1879; AVX1-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
1880; AVX1-NEXT:    vmovd %xmm0, %eax
1881; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm1[3,1,2,3]
1882; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1883; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
1884; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1885; AVX1-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
1886; AVX1-NEXT:    vmovd %xmm1, %eax
1887; AVX1-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm1
1888; AVX1-NEXT:    vmovd %xmm0, %eax
1889; AVX1-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
1890; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
1891; AVX1-NEXT:    retq
1892;
1893; AVX2-LABEL: cvt_16f32_to_16i16:
1894; AVX2:       # %bb.0:
1895; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm2
1896; AVX2-NEXT:    vmovd %xmm2, %eax
1897; AVX2-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
1898; AVX2-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
1899; AVX2-NEXT:    vmovd %eax, %xmm3
1900; AVX2-NEXT:    vmovd %xmm2, %eax
1901; AVX2-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
1902; AVX2-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
1903; AVX2-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
1904; AVX2-NEXT:    vmovd %xmm2, %eax
1905; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
1906; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
1907; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1908; AVX2-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
1909; AVX2-NEXT:    vmovd %xmm1, %eax
1910; AVX2-NEXT:    vcvtps2ph $4, %xmm2, %xmm1
1911; AVX2-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
1912; AVX2-NEXT:    vmovd %xmm1, %eax
1913; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
1914; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1915; AVX2-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
1916; AVX2-NEXT:    vmovd %xmm1, %eax
1917; AVX2-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm2[1,0]
1918; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1919; AVX2-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
1920; AVX2-NEXT:    vmovd %xmm1, %eax
1921; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
1922; AVX2-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
1923; AVX2-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
1924; AVX2-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm3
1925; AVX2-NEXT:    vmovd %xmm2, %eax
1926; AVX2-NEXT:    vpinsrw $7, %eax, %xmm3, %xmm2
1927; AVX2-NEXT:    vmovd %xmm1, %eax
1928; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
1929; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1930; AVX2-NEXT:    vmovd %eax, %xmm3
1931; AVX2-NEXT:    vmovd %xmm1, %eax
1932; AVX2-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
1933; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1934; AVX2-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
1935; AVX2-NEXT:    vmovd %xmm1, %eax
1936; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
1937; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
1938; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1939; AVX2-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
1940; AVX2-NEXT:    vmovd %xmm0, %eax
1941; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm0
1942; AVX2-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
1943; AVX2-NEXT:    vmovd %xmm0, %eax
1944; AVX2-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
1945; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1946; AVX2-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
1947; AVX2-NEXT:    vmovd %xmm0, %eax
1948; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm1[3,1,2,3]
1949; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
1950; AVX2-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
1951; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1952; AVX2-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
1953; AVX2-NEXT:    vmovd %xmm1, %eax
1954; AVX2-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm1
1955; AVX2-NEXT:    vmovd %xmm0, %eax
1956; AVX2-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
1957; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
1958; AVX2-NEXT:    retq
1959;
1960; AVX512-LABEL: cvt_16f32_to_16i16:
1961; AVX512:       # %bb.0:
1962; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
1963; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm2
1964; AVX512-NEXT:    vmovd %xmm2, %eax
1965; AVX512-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
1966; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
1967; AVX512-NEXT:    vmovd %eax, %xmm3
1968; AVX512-NEXT:    vmovd %xmm2, %eax
1969; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
1970; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
1971; AVX512-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
1972; AVX512-NEXT:    vmovd %xmm2, %eax
1973; AVX512-NEXT:    vextractf128 $1, %ymm1, %xmm2
1974; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
1975; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1976; AVX512-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
1977; AVX512-NEXT:    vmovd %xmm1, %eax
1978; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm1
1979; AVX512-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
1980; AVX512-NEXT:    vmovd %xmm1, %eax
1981; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
1982; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1983; AVX512-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
1984; AVX512-NEXT:    vmovd %xmm1, %eax
1985; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm2[1,0]
1986; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1987; AVX512-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
1988; AVX512-NEXT:    vmovd %xmm1, %eax
1989; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
1990; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
1991; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
1992; AVX512-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm3
1993; AVX512-NEXT:    vmovd %xmm2, %eax
1994; AVX512-NEXT:    vpinsrw $7, %eax, %xmm3, %xmm2
1995; AVX512-NEXT:    vmovd %xmm1, %eax
1996; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
1997; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
1998; AVX512-NEXT:    vmovd %eax, %xmm3
1999; AVX512-NEXT:    vmovd %xmm1, %eax
2000; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
2001; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2002; AVX512-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
2003; AVX512-NEXT:    vmovd %xmm1, %eax
2004; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
2005; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
2006; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2007; AVX512-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
2008; AVX512-NEXT:    vmovd %xmm0, %eax
2009; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm0
2010; AVX512-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
2011; AVX512-NEXT:    vmovd %xmm0, %eax
2012; AVX512-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
2013; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2014; AVX512-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
2015; AVX512-NEXT:    vmovd %xmm0, %eax
2016; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm1[1,0]
2017; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2018; AVX512-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
2019; AVX512-NEXT:    vmovd %xmm0, %eax
2020; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm1[3,1,2,3]
2021; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2022; AVX512-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm1
2023; AVX512-NEXT:    vmovd %xmm0, %eax
2024; AVX512-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
2025; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
2026; AVX512-NEXT:    retq
2027  %1 = fptrunc <16 x float> %a0 to <16 x half>
2028  %2 = bitcast <16 x half> %1 to <16 x i16>
2029  ret <16 x i16> %2
2030}
2031
2032;
2033; Float to Half (Store)
2034;
2035
2036define void @store_cvt_f32_to_i16(float %a0, i16* %a1) nounwind {
2037; ALL-LABEL: store_cvt_f32_to_i16:
2038; ALL:       # %bb.0:
2039; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2040; ALL-NEXT:    vmovd %xmm0, %eax
2041; ALL-NEXT:    movw %ax, (%rdi)
2042; ALL-NEXT:    retq
2043  %1 = fptrunc float %a0 to half
2044  %2 = bitcast half %1 to i16
2045  store i16 %2, i16* %a1
2046  ret void
2047}
2048
2049define void @store_cvt_4f32_to_4i16(<4 x float> %a0, <4 x i16>* %a1) nounwind {
2050; ALL-LABEL: store_cvt_4f32_to_4i16:
2051; ALL:       # %bb.0:
2052; ALL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2053; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2054; ALL-NEXT:    vmovd %xmm1, %eax
2055; ALL-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
2056; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2057; ALL-NEXT:    vmovd %xmm1, %ecx
2058; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2059; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2060; ALL-NEXT:    vmovd %xmm1, %edx
2061; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2062; ALL-NEXT:    vmovd %xmm0, %esi
2063; ALL-NEXT:    movw %si, (%rdi)
2064; ALL-NEXT:    movw %dx, 6(%rdi)
2065; ALL-NEXT:    movw %cx, 4(%rdi)
2066; ALL-NEXT:    movw %ax, 2(%rdi)
2067; ALL-NEXT:    retq
2068  %1 = fptrunc <4 x float> %a0 to <4 x half>
2069  %2 = bitcast <4 x half> %1 to <4 x i16>
2070  store <4 x i16> %2, <4 x i16>* %a1
2071  ret void
2072}
2073
2074define void @store_cvt_4f32_to_8i16_undef(<4 x float> %a0, <8 x i16>* %a1) nounwind {
2075; ALL-LABEL: store_cvt_4f32_to_8i16_undef:
2076; ALL:       # %bb.0:
2077; ALL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2078; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2079; ALL-NEXT:    vmovd %xmm1, %eax
2080; ALL-NEXT:    shll $16, %eax
2081; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2082; ALL-NEXT:    vmovd %xmm1, %ecx
2083; ALL-NEXT:    movzwl %cx, %ecx
2084; ALL-NEXT:    orl %eax, %ecx
2085; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2086; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2087; ALL-NEXT:    vmovd %xmm1, %eax
2088; ALL-NEXT:    shll $16, %eax
2089; ALL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2090; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2091; ALL-NEXT:    vmovd %xmm0, %edx
2092; ALL-NEXT:    movzwl %dx, %edx
2093; ALL-NEXT:    orl %eax, %edx
2094; ALL-NEXT:    shlq $32, %rdx
2095; ALL-NEXT:    orq %rcx, %rdx
2096; ALL-NEXT:    vmovq %rdx, %xmm0
2097; ALL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2098; ALL-NEXT:    vmovdqa %xmm0, (%rdi)
2099; ALL-NEXT:    retq
2100  %1 = fptrunc <4 x float> %a0 to <4 x half>
2101  %2 = bitcast <4 x half> %1 to <4 x i16>
2102  %3 = shufflevector <4 x i16> %2, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
2103  store <8 x i16> %3, <8 x i16>* %a1
2104  ret void
2105}
2106
2107define void @store_cvt_4f32_to_8i16_zero(<4 x float> %a0, <8 x i16>* %a1) nounwind {
2108; AVX1-LABEL: store_cvt_4f32_to_8i16_zero:
2109; AVX1:       # %bb.0:
2110; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2111; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2112; AVX1-NEXT:    vmovd %xmm1, %eax
2113; AVX1-NEXT:    shll $16, %eax
2114; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2115; AVX1-NEXT:    vmovd %xmm1, %ecx
2116; AVX1-NEXT:    movzwl %cx, %ecx
2117; AVX1-NEXT:    orl %eax, %ecx
2118; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2119; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2120; AVX1-NEXT:    vmovd %xmm1, %eax
2121; AVX1-NEXT:    shll $16, %eax
2122; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2123; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2124; AVX1-NEXT:    vmovd %xmm0, %edx
2125; AVX1-NEXT:    movzwl %dx, %edx
2126; AVX1-NEXT:    orl %eax, %edx
2127; AVX1-NEXT:    shlq $32, %rdx
2128; AVX1-NEXT:    orq %rcx, %rdx
2129; AVX1-NEXT:    vmovq %rdx, %xmm0
2130; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2131; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
2132; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
2133; AVX1-NEXT:    retq
2134;
2135; AVX2-SLOW-LABEL: store_cvt_4f32_to_8i16_zero:
2136; AVX2-SLOW:       # %bb.0:
2137; AVX2-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2138; AVX2-SLOW-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2139; AVX2-SLOW-NEXT:    vmovd %xmm1, %eax
2140; AVX2-SLOW-NEXT:    shll $16, %eax
2141; AVX2-SLOW-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2142; AVX2-SLOW-NEXT:    vmovd %xmm1, %ecx
2143; AVX2-SLOW-NEXT:    movzwl %cx, %ecx
2144; AVX2-SLOW-NEXT:    orl %eax, %ecx
2145; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2146; AVX2-SLOW-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2147; AVX2-SLOW-NEXT:    vmovd %xmm1, %eax
2148; AVX2-SLOW-NEXT:    shll $16, %eax
2149; AVX2-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2150; AVX2-SLOW-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2151; AVX2-SLOW-NEXT:    vmovd %xmm0, %edx
2152; AVX2-SLOW-NEXT:    movzwl %dx, %edx
2153; AVX2-SLOW-NEXT:    orl %eax, %edx
2154; AVX2-SLOW-NEXT:    shlq $32, %rdx
2155; AVX2-SLOW-NEXT:    orq %rcx, %rdx
2156; AVX2-SLOW-NEXT:    vmovq %rdx, %xmm0
2157; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2158; AVX2-SLOW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
2159; AVX2-SLOW-NEXT:    vmovdqa %xmm0, (%rdi)
2160; AVX2-SLOW-NEXT:    retq
2161;
2162; AVX2-FAST-LABEL: store_cvt_4f32_to_8i16_zero:
2163; AVX2-FAST:       # %bb.0:
2164; AVX2-FAST-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2165; AVX2-FAST-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2166; AVX2-FAST-NEXT:    vmovd %xmm1, %eax
2167; AVX2-FAST-NEXT:    shll $16, %eax
2168; AVX2-FAST-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2169; AVX2-FAST-NEXT:    vmovd %xmm1, %ecx
2170; AVX2-FAST-NEXT:    movzwl %cx, %ecx
2171; AVX2-FAST-NEXT:    orl %eax, %ecx
2172; AVX2-FAST-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2173; AVX2-FAST-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2174; AVX2-FAST-NEXT:    vmovd %xmm1, %eax
2175; AVX2-FAST-NEXT:    shll $16, %eax
2176; AVX2-FAST-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2177; AVX2-FAST-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2178; AVX2-FAST-NEXT:    vmovd %xmm0, %edx
2179; AVX2-FAST-NEXT:    movzwl %dx, %edx
2180; AVX2-FAST-NEXT:    orl %eax, %edx
2181; AVX2-FAST-NEXT:    shlq $32, %rdx
2182; AVX2-FAST-NEXT:    orq %rcx, %rdx
2183; AVX2-FAST-NEXT:    vmovq %rdx, %xmm0
2184; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero
2185; AVX2-FAST-NEXT:    vmovdqa %xmm0, (%rdi)
2186; AVX2-FAST-NEXT:    retq
2187;
2188; AVX512F-LABEL: store_cvt_4f32_to_8i16_zero:
2189; AVX512F:       # %bb.0:
2190; AVX512F-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2191; AVX512F-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2192; AVX512F-NEXT:    vmovd %xmm1, %eax
2193; AVX512F-NEXT:    shll $16, %eax
2194; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2195; AVX512F-NEXT:    vmovd %xmm1, %ecx
2196; AVX512F-NEXT:    movzwl %cx, %ecx
2197; AVX512F-NEXT:    orl %eax, %ecx
2198; AVX512F-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2199; AVX512F-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2200; AVX512F-NEXT:    vmovd %xmm1, %eax
2201; AVX512F-NEXT:    shll $16, %eax
2202; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2203; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2204; AVX512F-NEXT:    vmovd %xmm0, %edx
2205; AVX512F-NEXT:    movzwl %dx, %edx
2206; AVX512F-NEXT:    orl %eax, %edx
2207; AVX512F-NEXT:    shlq $32, %rdx
2208; AVX512F-NEXT:    orq %rcx, %rdx
2209; AVX512F-NEXT:    vmovq %rdx, %xmm0
2210; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2211; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
2212; AVX512F-NEXT:    vmovdqa %xmm0, (%rdi)
2213; AVX512F-NEXT:    retq
2214;
2215; AVX512VL-LABEL: store_cvt_4f32_to_8i16_zero:
2216; AVX512VL:       # %bb.0:
2217; AVX512VL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2218; AVX512VL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2219; AVX512VL-NEXT:    vmovd %xmm1, %eax
2220; AVX512VL-NEXT:    shll $16, %eax
2221; AVX512VL-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
2222; AVX512VL-NEXT:    vmovd %xmm1, %ecx
2223; AVX512VL-NEXT:    movzwl %cx, %ecx
2224; AVX512VL-NEXT:    orl %eax, %ecx
2225; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2226; AVX512VL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2227; AVX512VL-NEXT:    vmovd %xmm1, %eax
2228; AVX512VL-NEXT:    shll $16, %eax
2229; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2230; AVX512VL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2231; AVX512VL-NEXT:    vmovd %xmm0, %edx
2232; AVX512VL-NEXT:    movzwl %dx, %edx
2233; AVX512VL-NEXT:    orl %eax, %edx
2234; AVX512VL-NEXT:    shlq $32, %rdx
2235; AVX512VL-NEXT:    orq %rcx, %rdx
2236; AVX512VL-NEXT:    vmovq %rdx, %xmm0
2237; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero
2238; AVX512VL-NEXT:    vmovdqa %xmm0, (%rdi)
2239; AVX512VL-NEXT:    retq
2240  %1 = fptrunc <4 x float> %a0 to <4 x half>
2241  %2 = bitcast <4 x half> %1 to <4 x i16>
2242  %3 = shufflevector <4 x i16> %2, <4 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
2243  store <8 x i16> %3, <8 x i16>* %a1
2244  ret void
2245}
2246
2247define void @store_cvt_8f32_to_8i16(<8 x float> %a0, <8 x i16>* %a1) nounwind {
2248; ALL-LABEL: store_cvt_8f32_to_8i16:
2249; ALL:       # %bb.0:
2250; ALL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
2251; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2252; ALL-NEXT:    vmovd %xmm1, %r8d
2253; ALL-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
2254; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2255; ALL-NEXT:    vmovd %xmm1, %r9d
2256; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
2257; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2258; ALL-NEXT:    vmovd %xmm1, %r10d
2259; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm1
2260; ALL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
2261; ALL-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2262; ALL-NEXT:    vmovd %xmm2, %r11d
2263; ALL-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
2264; ALL-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2265; ALL-NEXT:    vmovd %xmm2, %eax
2266; ALL-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
2267; ALL-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2268; ALL-NEXT:    vmovd %xmm2, %ecx
2269; ALL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2270; ALL-NEXT:    vmovd %xmm0, %edx
2271; ALL-NEXT:    vcvtps2ph $4, %xmm1, %xmm0
2272; ALL-NEXT:    vmovd %xmm0, %esi
2273; ALL-NEXT:    movw %si, 8(%rdi)
2274; ALL-NEXT:    movw %dx, (%rdi)
2275; ALL-NEXT:    movw %cx, 14(%rdi)
2276; ALL-NEXT:    movw %ax, 12(%rdi)
2277; ALL-NEXT:    movw %r11w, 10(%rdi)
2278; ALL-NEXT:    movw %r10w, 6(%rdi)
2279; ALL-NEXT:    movw %r9w, 4(%rdi)
2280; ALL-NEXT:    movw %r8w, 2(%rdi)
2281; ALL-NEXT:    vzeroupper
2282; ALL-NEXT:    retq
2283  %1 = fptrunc <8 x float> %a0 to <8 x half>
2284  %2 = bitcast <8 x half> %1 to <8 x i16>
2285  store <8 x i16> %2, <8 x i16>* %a1
2286  ret void
2287}
2288
2289define void @store_cvt_16f32_to_16i16(<16 x float> %a0, <16 x i16>* %a1) nounwind {
2290; AVX1-LABEL: store_cvt_16f32_to_16i16:
2291; AVX1:       # %bb.0:
2292; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
2293; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
2294; AVX1-NEXT:    vcvtps2ph $4, %xmm3, %xmm4
2295; AVX1-NEXT:    vmovd %xmm4, %eax
2296; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm4
2297; AVX1-NEXT:    movw %ax, 24(%rdi)
2298; AVX1-NEXT:    vmovd %xmm4, %eax
2299; AVX1-NEXT:    vcvtps2ph $4, %xmm2, %xmm4
2300; AVX1-NEXT:    movw %ax, 16(%rdi)
2301; AVX1-NEXT:    vmovd %xmm4, %eax
2302; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm4
2303; AVX1-NEXT:    movw %ax, 8(%rdi)
2304; AVX1-NEXT:    vmovd %xmm4, %eax
2305; AVX1-NEXT:    vpermilps {{.*#+}} xmm4 = xmm3[3,1,2,3]
2306; AVX1-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2307; AVX1-NEXT:    movw %ax, (%rdi)
2308; AVX1-NEXT:    vmovd %xmm4, %eax
2309; AVX1-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm3[1,0]
2310; AVX1-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2311; AVX1-NEXT:    movw %ax, 30(%rdi)
2312; AVX1-NEXT:    vmovd %xmm4, %eax
2313; AVX1-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
2314; AVX1-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2315; AVX1-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm3[1,1,3,3]
2316; AVX1-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2317; AVX1-NEXT:    movw %ax, 28(%rdi)
2318; AVX1-NEXT:    vmovd %xmm3, %eax
2319; AVX1-NEXT:    vpermilps {{.*#+}} xmm3 = xmm1[3,1,2,3]
2320; AVX1-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2321; AVX1-NEXT:    movw %ax, 26(%rdi)
2322; AVX1-NEXT:    vmovd %xmm3, %eax
2323; AVX1-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
2324; AVX1-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2325; AVX1-NEXT:    movw %ax, 22(%rdi)
2326; AVX1-NEXT:    vmovd %xmm3, %eax
2327; AVX1-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
2328; AVX1-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2329; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
2330; AVX1-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2331; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm1[1,1,3,3]
2332; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2333; AVX1-NEXT:    movw %ax, 20(%rdi)
2334; AVX1-NEXT:    vmovd %xmm1, %eax
2335; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm2[3,1,2,3]
2336; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2337; AVX1-NEXT:    movw %ax, 18(%rdi)
2338; AVX1-NEXT:    vmovd %xmm1, %eax
2339; AVX1-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
2340; AVX1-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2341; AVX1-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm2[1,0]
2342; AVX1-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2343; AVX1-NEXT:    movw %ax, 14(%rdi)
2344; AVX1-NEXT:    vmovd %xmm2, %eax
2345; AVX1-NEXT:    movw %ax, 12(%rdi)
2346; AVX1-NEXT:    vmovd %xmm1, %eax
2347; AVX1-NEXT:    movw %ax, 10(%rdi)
2348; AVX1-NEXT:    vmovd %xmm0, %eax
2349; AVX1-NEXT:    movw %ax, 6(%rdi)
2350; AVX1-NEXT:    vmovd %xmm3, %eax
2351; AVX1-NEXT:    movw %ax, 4(%rdi)
2352; AVX1-NEXT:    vmovd %xmm4, %eax
2353; AVX1-NEXT:    movw %ax, 2(%rdi)
2354; AVX1-NEXT:    vzeroupper
2355; AVX1-NEXT:    retq
2356;
2357; AVX2-LABEL: store_cvt_16f32_to_16i16:
2358; AVX2:       # %bb.0:
2359; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm2
2360; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm3
2361; AVX2-NEXT:    vcvtps2ph $4, %xmm3, %xmm4
2362; AVX2-NEXT:    vmovd %xmm4, %eax
2363; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm4
2364; AVX2-NEXT:    movw %ax, 24(%rdi)
2365; AVX2-NEXT:    vmovd %xmm4, %eax
2366; AVX2-NEXT:    vcvtps2ph $4, %xmm2, %xmm4
2367; AVX2-NEXT:    movw %ax, 16(%rdi)
2368; AVX2-NEXT:    vmovd %xmm4, %eax
2369; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm4
2370; AVX2-NEXT:    movw %ax, 8(%rdi)
2371; AVX2-NEXT:    vmovd %xmm4, %eax
2372; AVX2-NEXT:    vpermilps {{.*#+}} xmm4 = xmm3[3,1,2,3]
2373; AVX2-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2374; AVX2-NEXT:    movw %ax, (%rdi)
2375; AVX2-NEXT:    vmovd %xmm4, %eax
2376; AVX2-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm3[1,0]
2377; AVX2-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2378; AVX2-NEXT:    movw %ax, 30(%rdi)
2379; AVX2-NEXT:    vmovd %xmm4, %eax
2380; AVX2-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
2381; AVX2-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2382; AVX2-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm3[1,1,3,3]
2383; AVX2-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2384; AVX2-NEXT:    movw %ax, 28(%rdi)
2385; AVX2-NEXT:    vmovd %xmm3, %eax
2386; AVX2-NEXT:    vpermilps {{.*#+}} xmm3 = xmm1[3,1,2,3]
2387; AVX2-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2388; AVX2-NEXT:    movw %ax, 26(%rdi)
2389; AVX2-NEXT:    vmovd %xmm3, %eax
2390; AVX2-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
2391; AVX2-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2392; AVX2-NEXT:    movw %ax, 22(%rdi)
2393; AVX2-NEXT:    vmovd %xmm3, %eax
2394; AVX2-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
2395; AVX2-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2396; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
2397; AVX2-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2398; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm1[1,1,3,3]
2399; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2400; AVX2-NEXT:    movw %ax, 20(%rdi)
2401; AVX2-NEXT:    vmovd %xmm1, %eax
2402; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm2[3,1,2,3]
2403; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2404; AVX2-NEXT:    movw %ax, 18(%rdi)
2405; AVX2-NEXT:    vmovd %xmm1, %eax
2406; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
2407; AVX2-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2408; AVX2-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm2[1,0]
2409; AVX2-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2410; AVX2-NEXT:    movw %ax, 14(%rdi)
2411; AVX2-NEXT:    vmovd %xmm2, %eax
2412; AVX2-NEXT:    movw %ax, 12(%rdi)
2413; AVX2-NEXT:    vmovd %xmm1, %eax
2414; AVX2-NEXT:    movw %ax, 10(%rdi)
2415; AVX2-NEXT:    vmovd %xmm0, %eax
2416; AVX2-NEXT:    movw %ax, 6(%rdi)
2417; AVX2-NEXT:    vmovd %xmm3, %eax
2418; AVX2-NEXT:    movw %ax, 4(%rdi)
2419; AVX2-NEXT:    vmovd %xmm4, %eax
2420; AVX2-NEXT:    movw %ax, 2(%rdi)
2421; AVX2-NEXT:    vzeroupper
2422; AVX2-NEXT:    retq
2423;
2424; AVX512-LABEL: store_cvt_16f32_to_16i16:
2425; AVX512:       # %bb.0:
2426; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
2427; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
2428; AVX512-NEXT:    vextractf128 $1, %ymm2, %xmm3
2429; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm4
2430; AVX512-NEXT:    vmovd %xmm4, %eax
2431; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm4
2432; AVX512-NEXT:    movw %ax, 24(%rdi)
2433; AVX512-NEXT:    vmovd %xmm4, %eax
2434; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm4
2435; AVX512-NEXT:    movw %ax, 16(%rdi)
2436; AVX512-NEXT:    vmovd %xmm4, %eax
2437; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm4
2438; AVX512-NEXT:    movw %ax, 8(%rdi)
2439; AVX512-NEXT:    vmovd %xmm4, %eax
2440; AVX512-NEXT:    vpermilps {{.*#+}} xmm4 = xmm3[3,1,2,3]
2441; AVX512-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2442; AVX512-NEXT:    movw %ax, (%rdi)
2443; AVX512-NEXT:    vmovd %xmm4, %eax
2444; AVX512-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm3[1,0]
2445; AVX512-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2446; AVX512-NEXT:    movw %ax, 30(%rdi)
2447; AVX512-NEXT:    vmovd %xmm4, %eax
2448; AVX512-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
2449; AVX512-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
2450; AVX512-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm3[1,1,3,3]
2451; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2452; AVX512-NEXT:    movw %ax, 28(%rdi)
2453; AVX512-NEXT:    vmovd %xmm3, %eax
2454; AVX512-NEXT:    vpermilps {{.*#+}} xmm3 = xmm2[3,1,2,3]
2455; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2456; AVX512-NEXT:    movw %ax, 26(%rdi)
2457; AVX512-NEXT:    vmovd %xmm3, %eax
2458; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
2459; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2460; AVX512-NEXT:    movw %ax, 22(%rdi)
2461; AVX512-NEXT:    vmovd %xmm3, %eax
2462; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
2463; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
2464; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
2465; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
2466; AVX512-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm2[1,1,3,3]
2467; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2468; AVX512-NEXT:    movw %ax, 20(%rdi)
2469; AVX512-NEXT:    vmovd %xmm2, %eax
2470; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
2471; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2472; AVX512-NEXT:    movw %ax, 18(%rdi)
2473; AVX512-NEXT:    vmovd %xmm2, %eax
2474; AVX512-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
2475; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
2476; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
2477; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
2478; AVX512-NEXT:    movw %ax, 14(%rdi)
2479; AVX512-NEXT:    vmovd %xmm1, %eax
2480; AVX512-NEXT:    movw %ax, 12(%rdi)
2481; AVX512-NEXT:    vmovd %xmm2, %eax
2482; AVX512-NEXT:    movw %ax, 10(%rdi)
2483; AVX512-NEXT:    vmovd %xmm0, %eax
2484; AVX512-NEXT:    movw %ax, 6(%rdi)
2485; AVX512-NEXT:    vmovd %xmm3, %eax
2486; AVX512-NEXT:    movw %ax, 4(%rdi)
2487; AVX512-NEXT:    vmovd %xmm4, %eax
2488; AVX512-NEXT:    movw %ax, 2(%rdi)
2489; AVX512-NEXT:    vzeroupper
2490; AVX512-NEXT:    retq
2491  %1 = fptrunc <16 x float> %a0 to <16 x half>
2492  %2 = bitcast <16 x half> %1 to <16 x i16>
2493  store <16 x i16> %2, <16 x i16>* %a1
2494  ret void
2495}
2496
2497;
2498; Double to Half
2499;
2500
2501define i16 @cvt_f64_to_i16(double %a0) nounwind {
2502; ALL-LABEL: cvt_f64_to_i16:
2503; ALL:       # %bb.0:
2504; ALL-NEXT:    jmp __truncdfhf2 # TAILCALL
2505  %1 = fptrunc double %a0 to half
2506  %2 = bitcast half %1 to i16
2507  ret i16 %2
2508}
2509
2510define <2 x i16> @cvt_2f64_to_2i16(<2 x double> %a0) nounwind {
2511; ALL-LABEL: cvt_2f64_to_2i16:
2512; ALL:       # %bb.0:
2513; ALL-NEXT:    pushq %rbx
2514; ALL-NEXT:    subq $16, %rsp
2515; ALL-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
2516; ALL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2517; ALL-NEXT:    callq __truncdfhf2
2518; ALL-NEXT:    movl %eax, %ebx
2519; ALL-NEXT:    shll $16, %ebx
2520; ALL-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
2521; ALL-NEXT:    callq __truncdfhf2
2522; ALL-NEXT:    movzwl %ax, %eax
2523; ALL-NEXT:    orl %ebx, %eax
2524; ALL-NEXT:    vmovd %eax, %xmm0
2525; ALL-NEXT:    addq $16, %rsp
2526; ALL-NEXT:    popq %rbx
2527; ALL-NEXT:    retq
2528  %1 = fptrunc <2 x double> %a0 to <2 x half>
2529  %2 = bitcast <2 x half> %1 to <2 x i16>
2530  ret <2 x i16> %2
2531}
2532
2533define <4 x i16> @cvt_4f64_to_4i16(<4 x double> %a0) nounwind {
2534; AVX1-LABEL: cvt_4f64_to_4i16:
2535; AVX1:       # %bb.0:
2536; AVX1-NEXT:    pushq %r14
2537; AVX1-NEXT:    pushq %rbx
2538; AVX1-NEXT:    subq $40, %rsp
2539; AVX1-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
2540; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2541; AVX1-NEXT:    vzeroupper
2542; AVX1-NEXT:    callq __truncdfhf2
2543; AVX1-NEXT:    movl %eax, %ebx
2544; AVX1-NEXT:    shll $16, %ebx
2545; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2546; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2547; AVX1-NEXT:    vzeroupper
2548; AVX1-NEXT:    callq __truncdfhf2
2549; AVX1-NEXT:    movzwl %ax, %r14d
2550; AVX1-NEXT:    orl %ebx, %r14d
2551; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2552; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
2553; AVX1-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
2554; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2555; AVX1-NEXT:    vzeroupper
2556; AVX1-NEXT:    callq __truncdfhf2
2557; AVX1-NEXT:    movl %eax, %ebx
2558; AVX1-NEXT:    shll $16, %ebx
2559; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
2560; AVX1-NEXT:    callq __truncdfhf2
2561; AVX1-NEXT:    movzwl %ax, %eax
2562; AVX1-NEXT:    orl %ebx, %eax
2563; AVX1-NEXT:    shlq $32, %rax
2564; AVX1-NEXT:    orq %r14, %rax
2565; AVX1-NEXT:    vmovq %rax, %xmm0
2566; AVX1-NEXT:    addq $40, %rsp
2567; AVX1-NEXT:    popq %rbx
2568; AVX1-NEXT:    popq %r14
2569; AVX1-NEXT:    retq
2570;
2571; AVX2-LABEL: cvt_4f64_to_4i16:
2572; AVX2:       # %bb.0:
2573; AVX2-NEXT:    pushq %r14
2574; AVX2-NEXT:    pushq %rbx
2575; AVX2-NEXT:    subq $40, %rsp
2576; AVX2-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
2577; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2578; AVX2-NEXT:    vzeroupper
2579; AVX2-NEXT:    callq __truncdfhf2
2580; AVX2-NEXT:    movl %eax, %ebx
2581; AVX2-NEXT:    shll $16, %ebx
2582; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2583; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2584; AVX2-NEXT:    vzeroupper
2585; AVX2-NEXT:    callq __truncdfhf2
2586; AVX2-NEXT:    movzwl %ax, %r14d
2587; AVX2-NEXT:    orl %ebx, %r14d
2588; AVX2-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
2589; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
2590; AVX2-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
2591; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2592; AVX2-NEXT:    vzeroupper
2593; AVX2-NEXT:    callq __truncdfhf2
2594; AVX2-NEXT:    movl %eax, %ebx
2595; AVX2-NEXT:    shll $16, %ebx
2596; AVX2-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
2597; AVX2-NEXT:    callq __truncdfhf2
2598; AVX2-NEXT:    movzwl %ax, %eax
2599; AVX2-NEXT:    orl %ebx, %eax
2600; AVX2-NEXT:    shlq $32, %rax
2601; AVX2-NEXT:    orq %r14, %rax
2602; AVX2-NEXT:    vmovq %rax, %xmm0
2603; AVX2-NEXT:    addq $40, %rsp
2604; AVX2-NEXT:    popq %rbx
2605; AVX2-NEXT:    popq %r14
2606; AVX2-NEXT:    retq
2607;
2608; AVX512-LABEL: cvt_4f64_to_4i16:
2609; AVX512:       # %bb.0:
2610; AVX512-NEXT:    pushq %r14
2611; AVX512-NEXT:    pushq %rbx
2612; AVX512-NEXT:    subq $40, %rsp
2613; AVX512-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
2614; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2615; AVX512-NEXT:    vzeroupper
2616; AVX512-NEXT:    callq __truncdfhf2
2617; AVX512-NEXT:    movl %eax, %ebx
2618; AVX512-NEXT:    shll $16, %ebx
2619; AVX512-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2620; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2621; AVX512-NEXT:    vzeroupper
2622; AVX512-NEXT:    callq __truncdfhf2
2623; AVX512-NEXT:    movzwl %ax, %r14d
2624; AVX512-NEXT:    orl %ebx, %r14d
2625; AVX512-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
2626; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
2627; AVX512-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
2628; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2629; AVX512-NEXT:    vzeroupper
2630; AVX512-NEXT:    callq __truncdfhf2
2631; AVX512-NEXT:    movl %eax, %ebx
2632; AVX512-NEXT:    shll $16, %ebx
2633; AVX512-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
2634; AVX512-NEXT:    callq __truncdfhf2
2635; AVX512-NEXT:    movzwl %ax, %eax
2636; AVX512-NEXT:    orl %ebx, %eax
2637; AVX512-NEXT:    shlq $32, %rax
2638; AVX512-NEXT:    orq %r14, %rax
2639; AVX512-NEXT:    vmovq %rax, %xmm0
2640; AVX512-NEXT:    addq $40, %rsp
2641; AVX512-NEXT:    popq %rbx
2642; AVX512-NEXT:    popq %r14
2643; AVX512-NEXT:    retq
2644  %1 = fptrunc <4 x double> %a0 to <4 x half>
2645  %2 = bitcast <4 x half> %1 to <4 x i16>
2646  ret <4 x i16> %2
2647}
2648
2649define <8 x i16> @cvt_4f64_to_8i16_undef(<4 x double> %a0) nounwind {
2650; AVX1-LABEL: cvt_4f64_to_8i16_undef:
2651; AVX1:       # %bb.0:
2652; AVX1-NEXT:    pushq %r14
2653; AVX1-NEXT:    pushq %rbx
2654; AVX1-NEXT:    subq $40, %rsp
2655; AVX1-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
2656; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2657; AVX1-NEXT:    vzeroupper
2658; AVX1-NEXT:    callq __truncdfhf2
2659; AVX1-NEXT:    movl %eax, %ebx
2660; AVX1-NEXT:    shll $16, %ebx
2661; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2662; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2663; AVX1-NEXT:    vzeroupper
2664; AVX1-NEXT:    callq __truncdfhf2
2665; AVX1-NEXT:    movzwl %ax, %r14d
2666; AVX1-NEXT:    orl %ebx, %r14d
2667; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2668; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
2669; AVX1-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
2670; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2671; AVX1-NEXT:    vzeroupper
2672; AVX1-NEXT:    callq __truncdfhf2
2673; AVX1-NEXT:    movl %eax, %ebx
2674; AVX1-NEXT:    shll $16, %ebx
2675; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
2676; AVX1-NEXT:    callq __truncdfhf2
2677; AVX1-NEXT:    movzwl %ax, %eax
2678; AVX1-NEXT:    orl %ebx, %eax
2679; AVX1-NEXT:    shlq $32, %rax
2680; AVX1-NEXT:    orq %r14, %rax
2681; AVX1-NEXT:    vmovq %rax, %xmm0
2682; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2683; AVX1-NEXT:    addq $40, %rsp
2684; AVX1-NEXT:    popq %rbx
2685; AVX1-NEXT:    popq %r14
2686; AVX1-NEXT:    retq
2687;
2688; AVX2-LABEL: cvt_4f64_to_8i16_undef:
2689; AVX2:       # %bb.0:
2690; AVX2-NEXT:    pushq %r14
2691; AVX2-NEXT:    pushq %rbx
2692; AVX2-NEXT:    subq $40, %rsp
2693; AVX2-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
2694; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2695; AVX2-NEXT:    vzeroupper
2696; AVX2-NEXT:    callq __truncdfhf2
2697; AVX2-NEXT:    movl %eax, %ebx
2698; AVX2-NEXT:    shll $16, %ebx
2699; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2700; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2701; AVX2-NEXT:    vzeroupper
2702; AVX2-NEXT:    callq __truncdfhf2
2703; AVX2-NEXT:    movzwl %ax, %r14d
2704; AVX2-NEXT:    orl %ebx, %r14d
2705; AVX2-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
2706; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
2707; AVX2-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
2708; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2709; AVX2-NEXT:    vzeroupper
2710; AVX2-NEXT:    callq __truncdfhf2
2711; AVX2-NEXT:    movl %eax, %ebx
2712; AVX2-NEXT:    shll $16, %ebx
2713; AVX2-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
2714; AVX2-NEXT:    callq __truncdfhf2
2715; AVX2-NEXT:    movzwl %ax, %eax
2716; AVX2-NEXT:    orl %ebx, %eax
2717; AVX2-NEXT:    shlq $32, %rax
2718; AVX2-NEXT:    orq %r14, %rax
2719; AVX2-NEXT:    vmovq %rax, %xmm0
2720; AVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2721; AVX2-NEXT:    addq $40, %rsp
2722; AVX2-NEXT:    popq %rbx
2723; AVX2-NEXT:    popq %r14
2724; AVX2-NEXT:    retq
2725;
2726; AVX512-LABEL: cvt_4f64_to_8i16_undef:
2727; AVX512:       # %bb.0:
2728; AVX512-NEXT:    pushq %r14
2729; AVX512-NEXT:    pushq %rbx
2730; AVX512-NEXT:    subq $40, %rsp
2731; AVX512-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
2732; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2733; AVX512-NEXT:    vzeroupper
2734; AVX512-NEXT:    callq __truncdfhf2
2735; AVX512-NEXT:    movl %eax, %ebx
2736; AVX512-NEXT:    shll $16, %ebx
2737; AVX512-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2738; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2739; AVX512-NEXT:    vzeroupper
2740; AVX512-NEXT:    callq __truncdfhf2
2741; AVX512-NEXT:    movzwl %ax, %r14d
2742; AVX512-NEXT:    orl %ebx, %r14d
2743; AVX512-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
2744; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
2745; AVX512-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
2746; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2747; AVX512-NEXT:    vzeroupper
2748; AVX512-NEXT:    callq __truncdfhf2
2749; AVX512-NEXT:    movl %eax, %ebx
2750; AVX512-NEXT:    shll $16, %ebx
2751; AVX512-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
2752; AVX512-NEXT:    callq __truncdfhf2
2753; AVX512-NEXT:    movzwl %ax, %eax
2754; AVX512-NEXT:    orl %ebx, %eax
2755; AVX512-NEXT:    shlq $32, %rax
2756; AVX512-NEXT:    orq %r14, %rax
2757; AVX512-NEXT:    vmovq %rax, %xmm0
2758; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2759; AVX512-NEXT:    addq $40, %rsp
2760; AVX512-NEXT:    popq %rbx
2761; AVX512-NEXT:    popq %r14
2762; AVX512-NEXT:    retq
2763  %1 = fptrunc <4 x double> %a0 to <4 x half>
2764  %2 = bitcast <4 x half> %1 to <4 x i16>
2765  %3 = shufflevector <4 x i16> %2, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
2766  ret <8 x i16> %3
2767}
2768
2769define <8 x i16> @cvt_4f64_to_8i16_zero(<4 x double> %a0) nounwind {
2770; AVX1-LABEL: cvt_4f64_to_8i16_zero:
2771; AVX1:       # %bb.0:
2772; AVX1-NEXT:    pushq %r14
2773; AVX1-NEXT:    pushq %rbx
2774; AVX1-NEXT:    subq $40, %rsp
2775; AVX1-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
2776; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2777; AVX1-NEXT:    vzeroupper
2778; AVX1-NEXT:    callq __truncdfhf2
2779; AVX1-NEXT:    movl %eax, %ebx
2780; AVX1-NEXT:    shll $16, %ebx
2781; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2782; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2783; AVX1-NEXT:    vzeroupper
2784; AVX1-NEXT:    callq __truncdfhf2
2785; AVX1-NEXT:    movzwl %ax, %r14d
2786; AVX1-NEXT:    orl %ebx, %r14d
2787; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2788; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
2789; AVX1-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
2790; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2791; AVX1-NEXT:    vzeroupper
2792; AVX1-NEXT:    callq __truncdfhf2
2793; AVX1-NEXT:    movl %eax, %ebx
2794; AVX1-NEXT:    shll $16, %ebx
2795; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
2796; AVX1-NEXT:    callq __truncdfhf2
2797; AVX1-NEXT:    movzwl %ax, %eax
2798; AVX1-NEXT:    orl %ebx, %eax
2799; AVX1-NEXT:    shlq $32, %rax
2800; AVX1-NEXT:    orq %r14, %rax
2801; AVX1-NEXT:    vmovq %rax, %xmm0
2802; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2803; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
2804; AVX1-NEXT:    addq $40, %rsp
2805; AVX1-NEXT:    popq %rbx
2806; AVX1-NEXT:    popq %r14
2807; AVX1-NEXT:    retq
2808;
2809; AVX2-SLOW-LABEL: cvt_4f64_to_8i16_zero:
2810; AVX2-SLOW:       # %bb.0:
2811; AVX2-SLOW-NEXT:    pushq %r14
2812; AVX2-SLOW-NEXT:    pushq %rbx
2813; AVX2-SLOW-NEXT:    subq $40, %rsp
2814; AVX2-SLOW-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
2815; AVX2-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2816; AVX2-SLOW-NEXT:    vzeroupper
2817; AVX2-SLOW-NEXT:    callq __truncdfhf2
2818; AVX2-SLOW-NEXT:    movl %eax, %ebx
2819; AVX2-SLOW-NEXT:    shll $16, %ebx
2820; AVX2-SLOW-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2821; AVX2-SLOW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2822; AVX2-SLOW-NEXT:    vzeroupper
2823; AVX2-SLOW-NEXT:    callq __truncdfhf2
2824; AVX2-SLOW-NEXT:    movzwl %ax, %r14d
2825; AVX2-SLOW-NEXT:    orl %ebx, %r14d
2826; AVX2-SLOW-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
2827; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm0
2828; AVX2-SLOW-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
2829; AVX2-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2830; AVX2-SLOW-NEXT:    vzeroupper
2831; AVX2-SLOW-NEXT:    callq __truncdfhf2
2832; AVX2-SLOW-NEXT:    movl %eax, %ebx
2833; AVX2-SLOW-NEXT:    shll $16, %ebx
2834; AVX2-SLOW-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
2835; AVX2-SLOW-NEXT:    callq __truncdfhf2
2836; AVX2-SLOW-NEXT:    movzwl %ax, %eax
2837; AVX2-SLOW-NEXT:    orl %ebx, %eax
2838; AVX2-SLOW-NEXT:    shlq $32, %rax
2839; AVX2-SLOW-NEXT:    orq %r14, %rax
2840; AVX2-SLOW-NEXT:    vmovq %rax, %xmm0
2841; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2842; AVX2-SLOW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
2843; AVX2-SLOW-NEXT:    addq $40, %rsp
2844; AVX2-SLOW-NEXT:    popq %rbx
2845; AVX2-SLOW-NEXT:    popq %r14
2846; AVX2-SLOW-NEXT:    retq
2847;
2848; AVX2-FAST-LABEL: cvt_4f64_to_8i16_zero:
2849; AVX2-FAST:       # %bb.0:
2850; AVX2-FAST-NEXT:    pushq %r14
2851; AVX2-FAST-NEXT:    pushq %rbx
2852; AVX2-FAST-NEXT:    subq $40, %rsp
2853; AVX2-FAST-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
2854; AVX2-FAST-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2855; AVX2-FAST-NEXT:    vzeroupper
2856; AVX2-FAST-NEXT:    callq __truncdfhf2
2857; AVX2-FAST-NEXT:    movl %eax, %ebx
2858; AVX2-FAST-NEXT:    shll $16, %ebx
2859; AVX2-FAST-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2860; AVX2-FAST-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2861; AVX2-FAST-NEXT:    vzeroupper
2862; AVX2-FAST-NEXT:    callq __truncdfhf2
2863; AVX2-FAST-NEXT:    movzwl %ax, %r14d
2864; AVX2-FAST-NEXT:    orl %ebx, %r14d
2865; AVX2-FAST-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
2866; AVX2-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm0
2867; AVX2-FAST-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
2868; AVX2-FAST-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2869; AVX2-FAST-NEXT:    vzeroupper
2870; AVX2-FAST-NEXT:    callq __truncdfhf2
2871; AVX2-FAST-NEXT:    movl %eax, %ebx
2872; AVX2-FAST-NEXT:    shll $16, %ebx
2873; AVX2-FAST-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
2874; AVX2-FAST-NEXT:    callq __truncdfhf2
2875; AVX2-FAST-NEXT:    movzwl %ax, %eax
2876; AVX2-FAST-NEXT:    orl %ebx, %eax
2877; AVX2-FAST-NEXT:    shlq $32, %rax
2878; AVX2-FAST-NEXT:    orq %r14, %rax
2879; AVX2-FAST-NEXT:    vmovq %rax, %xmm0
2880; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero
2881; AVX2-FAST-NEXT:    addq $40, %rsp
2882; AVX2-FAST-NEXT:    popq %rbx
2883; AVX2-FAST-NEXT:    popq %r14
2884; AVX2-FAST-NEXT:    retq
2885;
2886; AVX512F-LABEL: cvt_4f64_to_8i16_zero:
2887; AVX512F:       # %bb.0:
2888; AVX512F-NEXT:    pushq %r14
2889; AVX512F-NEXT:    pushq %rbx
2890; AVX512F-NEXT:    subq $40, %rsp
2891; AVX512F-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
2892; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2893; AVX512F-NEXT:    vzeroupper
2894; AVX512F-NEXT:    callq __truncdfhf2
2895; AVX512F-NEXT:    movl %eax, %ebx
2896; AVX512F-NEXT:    shll $16, %ebx
2897; AVX512F-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2898; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2899; AVX512F-NEXT:    vzeroupper
2900; AVX512F-NEXT:    callq __truncdfhf2
2901; AVX512F-NEXT:    movzwl %ax, %r14d
2902; AVX512F-NEXT:    orl %ebx, %r14d
2903; AVX512F-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
2904; AVX512F-NEXT:    vextractf128 $1, %ymm0, %xmm0
2905; AVX512F-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
2906; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2907; AVX512F-NEXT:    vzeroupper
2908; AVX512F-NEXT:    callq __truncdfhf2
2909; AVX512F-NEXT:    movl %eax, %ebx
2910; AVX512F-NEXT:    shll $16, %ebx
2911; AVX512F-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
2912; AVX512F-NEXT:    callq __truncdfhf2
2913; AVX512F-NEXT:    movzwl %ax, %eax
2914; AVX512F-NEXT:    orl %ebx, %eax
2915; AVX512F-NEXT:    shlq $32, %rax
2916; AVX512F-NEXT:    orq %r14, %rax
2917; AVX512F-NEXT:    vmovq %rax, %xmm0
2918; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2919; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
2920; AVX512F-NEXT:    addq $40, %rsp
2921; AVX512F-NEXT:    popq %rbx
2922; AVX512F-NEXT:    popq %r14
2923; AVX512F-NEXT:    retq
2924;
2925; AVX512VL-LABEL: cvt_4f64_to_8i16_zero:
2926; AVX512VL:       # %bb.0:
2927; AVX512VL-NEXT:    pushq %r14
2928; AVX512VL-NEXT:    pushq %rbx
2929; AVX512VL-NEXT:    subq $40, %rsp
2930; AVX512VL-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
2931; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2932; AVX512VL-NEXT:    vzeroupper
2933; AVX512VL-NEXT:    callq __truncdfhf2
2934; AVX512VL-NEXT:    movl %eax, %ebx
2935; AVX512VL-NEXT:    shll $16, %ebx
2936; AVX512VL-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
2937; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2938; AVX512VL-NEXT:    vzeroupper
2939; AVX512VL-NEXT:    callq __truncdfhf2
2940; AVX512VL-NEXT:    movzwl %ax, %r14d
2941; AVX512VL-NEXT:    orl %ebx, %r14d
2942; AVX512VL-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
2943; AVX512VL-NEXT:    vextractf128 $1, %ymm0, %xmm0
2944; AVX512VL-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
2945; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2946; AVX512VL-NEXT:    vzeroupper
2947; AVX512VL-NEXT:    callq __truncdfhf2
2948; AVX512VL-NEXT:    movl %eax, %ebx
2949; AVX512VL-NEXT:    shll $16, %ebx
2950; AVX512VL-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
2951; AVX512VL-NEXT:    callq __truncdfhf2
2952; AVX512VL-NEXT:    movzwl %ax, %eax
2953; AVX512VL-NEXT:    orl %ebx, %eax
2954; AVX512VL-NEXT:    shlq $32, %rax
2955; AVX512VL-NEXT:    orq %r14, %rax
2956; AVX512VL-NEXT:    vmovq %rax, %xmm0
2957; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero
2958; AVX512VL-NEXT:    addq $40, %rsp
2959; AVX512VL-NEXT:    popq %rbx
2960; AVX512VL-NEXT:    popq %r14
2961; AVX512VL-NEXT:    retq
2962  %1 = fptrunc <4 x double> %a0 to <4 x half>
2963  %2 = bitcast <4 x half> %1 to <4 x i16>
2964  %3 = shufflevector <4 x i16> %2, <4 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
2965  ret <8 x i16> %3
2966}
2967
2968define <8 x i16> @cvt_8f64_to_8i16(<8 x double> %a0) nounwind {
2969; AVX1-LABEL: cvt_8f64_to_8i16:
2970; AVX1:       # %bb.0:
2971; AVX1-NEXT:    pushq %r15
2972; AVX1-NEXT:    pushq %r14
2973; AVX1-NEXT:    pushq %rbx
2974; AVX1-NEXT:    subq $64, %rsp
2975; AVX1-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
2976; AVX1-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
2977; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2978; AVX1-NEXT:    vzeroupper
2979; AVX1-NEXT:    callq __truncdfhf2
2980; AVX1-NEXT:    movl %eax, %ebx
2981; AVX1-NEXT:    shll $16, %ebx
2982; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
2983; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
2984; AVX1-NEXT:    vzeroupper
2985; AVX1-NEXT:    callq __truncdfhf2
2986; AVX1-NEXT:    movzwl %ax, %r15d
2987; AVX1-NEXT:    orl %ebx, %r15d
2988; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
2989; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
2990; AVX1-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2991; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
2992; AVX1-NEXT:    vzeroupper
2993; AVX1-NEXT:    callq __truncdfhf2
2994; AVX1-NEXT:    movl %eax, %ebx
2995; AVX1-NEXT:    shll $16, %ebx
2996; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
2997; AVX1-NEXT:    callq __truncdfhf2
2998; AVX1-NEXT:    movzwl %ax, %r14d
2999; AVX1-NEXT:    orl %ebx, %r14d
3000; AVX1-NEXT:    shlq $32, %r14
3001; AVX1-NEXT:    orq %r15, %r14
3002; AVX1-NEXT:    vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload
3003; AVX1-NEXT:    # xmm0 = mem[1,0]
3004; AVX1-NEXT:    callq __truncdfhf2
3005; AVX1-NEXT:    movl %eax, %ebx
3006; AVX1-NEXT:    shll $16, %ebx
3007; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3008; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3009; AVX1-NEXT:    vzeroupper
3010; AVX1-NEXT:    callq __truncdfhf2
3011; AVX1-NEXT:    movzwl %ax, %r15d
3012; AVX1-NEXT:    orl %ebx, %r15d
3013; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3014; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3015; AVX1-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3016; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3017; AVX1-NEXT:    vzeroupper
3018; AVX1-NEXT:    callq __truncdfhf2
3019; AVX1-NEXT:    movl %eax, %ebx
3020; AVX1-NEXT:    shll $16, %ebx
3021; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3022; AVX1-NEXT:    callq __truncdfhf2
3023; AVX1-NEXT:    movzwl %ax, %eax
3024; AVX1-NEXT:    orl %ebx, %eax
3025; AVX1-NEXT:    shlq $32, %rax
3026; AVX1-NEXT:    orq %r15, %rax
3027; AVX1-NEXT:    vmovq %rax, %xmm0
3028; AVX1-NEXT:    vmovq %r14, %xmm1
3029; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
3030; AVX1-NEXT:    addq $64, %rsp
3031; AVX1-NEXT:    popq %rbx
3032; AVX1-NEXT:    popq %r14
3033; AVX1-NEXT:    popq %r15
3034; AVX1-NEXT:    retq
3035;
3036; AVX2-LABEL: cvt_8f64_to_8i16:
3037; AVX2:       # %bb.0:
3038; AVX2-NEXT:    pushq %r15
3039; AVX2-NEXT:    pushq %r14
3040; AVX2-NEXT:    pushq %rbx
3041; AVX2-NEXT:    subq $64, %rsp
3042; AVX2-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
3043; AVX2-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3044; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3045; AVX2-NEXT:    vzeroupper
3046; AVX2-NEXT:    callq __truncdfhf2
3047; AVX2-NEXT:    movl %eax, %ebx
3048; AVX2-NEXT:    shll $16, %ebx
3049; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3050; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3051; AVX2-NEXT:    vzeroupper
3052; AVX2-NEXT:    callq __truncdfhf2
3053; AVX2-NEXT:    movzwl %ax, %r15d
3054; AVX2-NEXT:    orl %ebx, %r15d
3055; AVX2-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3056; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
3057; AVX2-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3058; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3059; AVX2-NEXT:    vzeroupper
3060; AVX2-NEXT:    callq __truncdfhf2
3061; AVX2-NEXT:    movl %eax, %ebx
3062; AVX2-NEXT:    shll $16, %ebx
3063; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3064; AVX2-NEXT:    callq __truncdfhf2
3065; AVX2-NEXT:    movzwl %ax, %r14d
3066; AVX2-NEXT:    orl %ebx, %r14d
3067; AVX2-NEXT:    shlq $32, %r14
3068; AVX2-NEXT:    orq %r15, %r14
3069; AVX2-NEXT:    vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload
3070; AVX2-NEXT:    # xmm0 = mem[1,0]
3071; AVX2-NEXT:    callq __truncdfhf2
3072; AVX2-NEXT:    movl %eax, %ebx
3073; AVX2-NEXT:    shll $16, %ebx
3074; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3075; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3076; AVX2-NEXT:    vzeroupper
3077; AVX2-NEXT:    callq __truncdfhf2
3078; AVX2-NEXT:    movzwl %ax, %r15d
3079; AVX2-NEXT:    orl %ebx, %r15d
3080; AVX2-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3081; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
3082; AVX2-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3083; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3084; AVX2-NEXT:    vzeroupper
3085; AVX2-NEXT:    callq __truncdfhf2
3086; AVX2-NEXT:    movl %eax, %ebx
3087; AVX2-NEXT:    shll $16, %ebx
3088; AVX2-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3089; AVX2-NEXT:    callq __truncdfhf2
3090; AVX2-NEXT:    movzwl %ax, %eax
3091; AVX2-NEXT:    orl %ebx, %eax
3092; AVX2-NEXT:    shlq $32, %rax
3093; AVX2-NEXT:    orq %r15, %rax
3094; AVX2-NEXT:    vmovq %rax, %xmm0
3095; AVX2-NEXT:    vmovq %r14, %xmm1
3096; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
3097; AVX2-NEXT:    addq $64, %rsp
3098; AVX2-NEXT:    popq %rbx
3099; AVX2-NEXT:    popq %r14
3100; AVX2-NEXT:    popq %r15
3101; AVX2-NEXT:    retq
3102;
3103; AVX512-LABEL: cvt_8f64_to_8i16:
3104; AVX512:       # %bb.0:
3105; AVX512-NEXT:    pushq %r15
3106; AVX512-NEXT:    pushq %r14
3107; AVX512-NEXT:    pushq %rbx
3108; AVX512-NEXT:    subq $96, %rsp
3109; AVX512-NEXT:    vmovupd %zmm0, (%rsp) # 64-byte Spill
3110; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3111; AVX512-NEXT:    vzeroupper
3112; AVX512-NEXT:    callq __truncdfhf2
3113; AVX512-NEXT:    movl %eax, %ebx
3114; AVX512-NEXT:    shll $16, %ebx
3115; AVX512-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
3116; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
3117; AVX512-NEXT:    vzeroupper
3118; AVX512-NEXT:    callq __truncdfhf2
3119; AVX512-NEXT:    movzwl %ax, %r15d
3120; AVX512-NEXT:    orl %ebx, %r15d
3121; AVX512-NEXT:    vmovupd (%rsp), %zmm0 # 64-byte Reload
3122; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
3123; AVX512-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3124; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3125; AVX512-NEXT:    vzeroupper
3126; AVX512-NEXT:    callq __truncdfhf2
3127; AVX512-NEXT:    movl %eax, %ebx
3128; AVX512-NEXT:    shll $16, %ebx
3129; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3130; AVX512-NEXT:    callq __truncdfhf2
3131; AVX512-NEXT:    movzwl %ax, %r14d
3132; AVX512-NEXT:    orl %ebx, %r14d
3133; AVX512-NEXT:    shlq $32, %r14
3134; AVX512-NEXT:    orq %r15, %r14
3135; AVX512-NEXT:    vmovupd (%rsp), %zmm0 # 64-byte Reload
3136; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
3137; AVX512-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3138; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3139; AVX512-NEXT:    vzeroupper
3140; AVX512-NEXT:    callq __truncdfhf2
3141; AVX512-NEXT:    movl %eax, %ebx
3142; AVX512-NEXT:    shll $16, %ebx
3143; AVX512-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3144; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3145; AVX512-NEXT:    vzeroupper
3146; AVX512-NEXT:    callq __truncdfhf2
3147; AVX512-NEXT:    movzwl %ax, %r15d
3148; AVX512-NEXT:    orl %ebx, %r15d
3149; AVX512-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3150; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
3151; AVX512-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3152; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3153; AVX512-NEXT:    vzeroupper
3154; AVX512-NEXT:    callq __truncdfhf2
3155; AVX512-NEXT:    movl %eax, %ebx
3156; AVX512-NEXT:    shll $16, %ebx
3157; AVX512-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3158; AVX512-NEXT:    callq __truncdfhf2
3159; AVX512-NEXT:    movzwl %ax, %eax
3160; AVX512-NEXT:    orl %ebx, %eax
3161; AVX512-NEXT:    shlq $32, %rax
3162; AVX512-NEXT:    orq %r15, %rax
3163; AVX512-NEXT:    vmovq %rax, %xmm0
3164; AVX512-NEXT:    vmovq %r14, %xmm1
3165; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
3166; AVX512-NEXT:    addq $96, %rsp
3167; AVX512-NEXT:    popq %rbx
3168; AVX512-NEXT:    popq %r14
3169; AVX512-NEXT:    popq %r15
3170; AVX512-NEXT:    retq
3171  %1 = fptrunc <8 x double> %a0 to <8 x half>
3172  %2 = bitcast <8 x half> %1 to <8 x i16>
3173  ret <8 x i16> %2
3174}
3175
3176;
3177; Double to Half (Store)
3178;
3179
3180define void @store_cvt_f64_to_i16(double %a0, i16* %a1) nounwind {
3181; ALL-LABEL: store_cvt_f64_to_i16:
3182; ALL:       # %bb.0:
3183; ALL-NEXT:    pushq %rbx
3184; ALL-NEXT:    movq %rdi, %rbx
3185; ALL-NEXT:    callq __truncdfhf2
3186; ALL-NEXT:    movw %ax, (%rbx)
3187; ALL-NEXT:    popq %rbx
3188; ALL-NEXT:    retq
3189  %1 = fptrunc double %a0 to half
3190  %2 = bitcast half %1 to i16
3191  store i16 %2, i16* %a1
3192  ret void
3193}
3194
3195define void @store_cvt_2f64_to_2i16(<2 x double> %a0, <2 x i16>* %a1) nounwind {
3196; ALL-LABEL: store_cvt_2f64_to_2i16:
3197; ALL:       # %bb.0:
3198; ALL-NEXT:    pushq %rbp
3199; ALL-NEXT:    pushq %rbx
3200; ALL-NEXT:    subq $24, %rsp
3201; ALL-NEXT:    movq %rdi, %rbx
3202; ALL-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3203; ALL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3204; ALL-NEXT:    callq __truncdfhf2
3205; ALL-NEXT:    movl %eax, %ebp
3206; ALL-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3207; ALL-NEXT:    callq __truncdfhf2
3208; ALL-NEXT:    movw %ax, (%rbx)
3209; ALL-NEXT:    movw %bp, 2(%rbx)
3210; ALL-NEXT:    addq $24, %rsp
3211; ALL-NEXT:    popq %rbx
3212; ALL-NEXT:    popq %rbp
3213; ALL-NEXT:    retq
3214  %1 = fptrunc <2 x double> %a0 to <2 x half>
3215  %2 = bitcast <2 x half> %1 to <2 x i16>
3216  store <2 x i16> %2, <2 x i16>* %a1
3217  ret void
3218}
3219
3220define void @store_cvt_4f64_to_4i16(<4 x double> %a0, <4 x i16>* %a1) nounwind {
3221; AVX1-LABEL: store_cvt_4f64_to_4i16:
3222; AVX1:       # %bb.0:
3223; AVX1-NEXT:    pushq %rbp
3224; AVX1-NEXT:    pushq %r15
3225; AVX1-NEXT:    pushq %r14
3226; AVX1-NEXT:    pushq %rbx
3227; AVX1-NEXT:    subq $88, %rsp
3228; AVX1-NEXT:    movq %rdi, %rbx
3229; AVX1-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3230; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3231; AVX1-NEXT:    vzeroupper
3232; AVX1-NEXT:    callq __truncdfhf2
3233; AVX1-NEXT:    movl %eax, %r14d
3234; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3235; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3236; AVX1-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3237; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3238; AVX1-NEXT:    vzeroupper
3239; AVX1-NEXT:    callq __truncdfhf2
3240; AVX1-NEXT:    movl %eax, %r15d
3241; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3242; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3243; AVX1-NEXT:    vzeroupper
3244; AVX1-NEXT:    callq __truncdfhf2
3245; AVX1-NEXT:    movl %eax, %ebp
3246; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3247; AVX1-NEXT:    callq __truncdfhf2
3248; AVX1-NEXT:    movw %ax, 4(%rbx)
3249; AVX1-NEXT:    movw %bp, (%rbx)
3250; AVX1-NEXT:    movw %r15w, 6(%rbx)
3251; AVX1-NEXT:    movw %r14w, 2(%rbx)
3252; AVX1-NEXT:    addq $88, %rsp
3253; AVX1-NEXT:    popq %rbx
3254; AVX1-NEXT:    popq %r14
3255; AVX1-NEXT:    popq %r15
3256; AVX1-NEXT:    popq %rbp
3257; AVX1-NEXT:    retq
3258;
3259; AVX2-LABEL: store_cvt_4f64_to_4i16:
3260; AVX2:       # %bb.0:
3261; AVX2-NEXT:    pushq %rbp
3262; AVX2-NEXT:    pushq %r15
3263; AVX2-NEXT:    pushq %r14
3264; AVX2-NEXT:    pushq %rbx
3265; AVX2-NEXT:    subq $88, %rsp
3266; AVX2-NEXT:    movq %rdi, %rbx
3267; AVX2-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3268; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3269; AVX2-NEXT:    vzeroupper
3270; AVX2-NEXT:    callq __truncdfhf2
3271; AVX2-NEXT:    movl %eax, %r14d
3272; AVX2-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3273; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
3274; AVX2-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3275; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3276; AVX2-NEXT:    vzeroupper
3277; AVX2-NEXT:    callq __truncdfhf2
3278; AVX2-NEXT:    movl %eax, %r15d
3279; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3280; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3281; AVX2-NEXT:    vzeroupper
3282; AVX2-NEXT:    callq __truncdfhf2
3283; AVX2-NEXT:    movl %eax, %ebp
3284; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3285; AVX2-NEXT:    callq __truncdfhf2
3286; AVX2-NEXT:    movw %ax, 4(%rbx)
3287; AVX2-NEXT:    movw %bp, (%rbx)
3288; AVX2-NEXT:    movw %r15w, 6(%rbx)
3289; AVX2-NEXT:    movw %r14w, 2(%rbx)
3290; AVX2-NEXT:    addq $88, %rsp
3291; AVX2-NEXT:    popq %rbx
3292; AVX2-NEXT:    popq %r14
3293; AVX2-NEXT:    popq %r15
3294; AVX2-NEXT:    popq %rbp
3295; AVX2-NEXT:    retq
3296;
3297; AVX512-LABEL: store_cvt_4f64_to_4i16:
3298; AVX512:       # %bb.0:
3299; AVX512-NEXT:    pushq %rbp
3300; AVX512-NEXT:    pushq %r15
3301; AVX512-NEXT:    pushq %r14
3302; AVX512-NEXT:    pushq %rbx
3303; AVX512-NEXT:    subq $88, %rsp
3304; AVX512-NEXT:    movq %rdi, %rbx
3305; AVX512-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3306; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3307; AVX512-NEXT:    vzeroupper
3308; AVX512-NEXT:    callq __truncdfhf2
3309; AVX512-NEXT:    movl %eax, %r14d
3310; AVX512-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3311; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
3312; AVX512-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3313; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3314; AVX512-NEXT:    vzeroupper
3315; AVX512-NEXT:    callq __truncdfhf2
3316; AVX512-NEXT:    movl %eax, %r15d
3317; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3318; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3319; AVX512-NEXT:    vzeroupper
3320; AVX512-NEXT:    callq __truncdfhf2
3321; AVX512-NEXT:    movl %eax, %ebp
3322; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3323; AVX512-NEXT:    callq __truncdfhf2
3324; AVX512-NEXT:    movw %ax, 4(%rbx)
3325; AVX512-NEXT:    movw %bp, (%rbx)
3326; AVX512-NEXT:    movw %r15w, 6(%rbx)
3327; AVX512-NEXT:    movw %r14w, 2(%rbx)
3328; AVX512-NEXT:    addq $88, %rsp
3329; AVX512-NEXT:    popq %rbx
3330; AVX512-NEXT:    popq %r14
3331; AVX512-NEXT:    popq %r15
3332; AVX512-NEXT:    popq %rbp
3333; AVX512-NEXT:    retq
3334  %1 = fptrunc <4 x double> %a0 to <4 x half>
3335  %2 = bitcast <4 x half> %1 to <4 x i16>
3336  store <4 x i16> %2, <4 x i16>* %a1
3337  ret void
3338}
3339
3340define void @store_cvt_4f64_to_8i16_undef(<4 x double> %a0, <8 x i16>* %a1) nounwind {
3341; AVX1-LABEL: store_cvt_4f64_to_8i16_undef:
3342; AVX1:       # %bb.0:
3343; AVX1-NEXT:    pushq %rbp
3344; AVX1-NEXT:    pushq %r14
3345; AVX1-NEXT:    pushq %rbx
3346; AVX1-NEXT:    subq $32, %rsp
3347; AVX1-NEXT:    movq %rdi, %r14
3348; AVX1-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3349; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3350; AVX1-NEXT:    vzeroupper
3351; AVX1-NEXT:    callq __truncdfhf2
3352; AVX1-NEXT:    movl %eax, %ebp
3353; AVX1-NEXT:    shll $16, %ebp
3354; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3355; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3356; AVX1-NEXT:    vzeroupper
3357; AVX1-NEXT:    callq __truncdfhf2
3358; AVX1-NEXT:    movzwl %ax, %ebx
3359; AVX1-NEXT:    orl %ebp, %ebx
3360; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3361; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3362; AVX1-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3363; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3364; AVX1-NEXT:    vzeroupper
3365; AVX1-NEXT:    callq __truncdfhf2
3366; AVX1-NEXT:    movl %eax, %ebp
3367; AVX1-NEXT:    shll $16, %ebp
3368; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3369; AVX1-NEXT:    callq __truncdfhf2
3370; AVX1-NEXT:    movzwl %ax, %eax
3371; AVX1-NEXT:    orl %ebp, %eax
3372; AVX1-NEXT:    shlq $32, %rax
3373; AVX1-NEXT:    orq %rbx, %rax
3374; AVX1-NEXT:    vmovq %rax, %xmm0
3375; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
3376; AVX1-NEXT:    vmovdqa %xmm0, (%r14)
3377; AVX1-NEXT:    addq $32, %rsp
3378; AVX1-NEXT:    popq %rbx
3379; AVX1-NEXT:    popq %r14
3380; AVX1-NEXT:    popq %rbp
3381; AVX1-NEXT:    retq
3382;
3383; AVX2-LABEL: store_cvt_4f64_to_8i16_undef:
3384; AVX2:       # %bb.0:
3385; AVX2-NEXT:    pushq %rbp
3386; AVX2-NEXT:    pushq %r14
3387; AVX2-NEXT:    pushq %rbx
3388; AVX2-NEXT:    subq $32, %rsp
3389; AVX2-NEXT:    movq %rdi, %r14
3390; AVX2-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3391; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3392; AVX2-NEXT:    vzeroupper
3393; AVX2-NEXT:    callq __truncdfhf2
3394; AVX2-NEXT:    movl %eax, %ebp
3395; AVX2-NEXT:    shll $16, %ebp
3396; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3397; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3398; AVX2-NEXT:    vzeroupper
3399; AVX2-NEXT:    callq __truncdfhf2
3400; AVX2-NEXT:    movzwl %ax, %ebx
3401; AVX2-NEXT:    orl %ebp, %ebx
3402; AVX2-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3403; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
3404; AVX2-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3405; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3406; AVX2-NEXT:    vzeroupper
3407; AVX2-NEXT:    callq __truncdfhf2
3408; AVX2-NEXT:    movl %eax, %ebp
3409; AVX2-NEXT:    shll $16, %ebp
3410; AVX2-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3411; AVX2-NEXT:    callq __truncdfhf2
3412; AVX2-NEXT:    movzwl %ax, %eax
3413; AVX2-NEXT:    orl %ebp, %eax
3414; AVX2-NEXT:    shlq $32, %rax
3415; AVX2-NEXT:    orq %rbx, %rax
3416; AVX2-NEXT:    vmovq %rax, %xmm0
3417; AVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
3418; AVX2-NEXT:    vmovdqa %xmm0, (%r14)
3419; AVX2-NEXT:    addq $32, %rsp
3420; AVX2-NEXT:    popq %rbx
3421; AVX2-NEXT:    popq %r14
3422; AVX2-NEXT:    popq %rbp
3423; AVX2-NEXT:    retq
3424;
3425; AVX512-LABEL: store_cvt_4f64_to_8i16_undef:
3426; AVX512:       # %bb.0:
3427; AVX512-NEXT:    pushq %rbp
3428; AVX512-NEXT:    pushq %r14
3429; AVX512-NEXT:    pushq %rbx
3430; AVX512-NEXT:    subq $32, %rsp
3431; AVX512-NEXT:    movq %rdi, %r14
3432; AVX512-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3433; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3434; AVX512-NEXT:    vzeroupper
3435; AVX512-NEXT:    callq __truncdfhf2
3436; AVX512-NEXT:    movl %eax, %ebp
3437; AVX512-NEXT:    shll $16, %ebp
3438; AVX512-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3439; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3440; AVX512-NEXT:    vzeroupper
3441; AVX512-NEXT:    callq __truncdfhf2
3442; AVX512-NEXT:    movzwl %ax, %ebx
3443; AVX512-NEXT:    orl %ebp, %ebx
3444; AVX512-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3445; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
3446; AVX512-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3447; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3448; AVX512-NEXT:    vzeroupper
3449; AVX512-NEXT:    callq __truncdfhf2
3450; AVX512-NEXT:    movl %eax, %ebp
3451; AVX512-NEXT:    shll $16, %ebp
3452; AVX512-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3453; AVX512-NEXT:    callq __truncdfhf2
3454; AVX512-NEXT:    movzwl %ax, %eax
3455; AVX512-NEXT:    orl %ebp, %eax
3456; AVX512-NEXT:    shlq $32, %rax
3457; AVX512-NEXT:    orq %rbx, %rax
3458; AVX512-NEXT:    vmovq %rax, %xmm0
3459; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
3460; AVX512-NEXT:    vmovdqa %xmm0, (%r14)
3461; AVX512-NEXT:    addq $32, %rsp
3462; AVX512-NEXT:    popq %rbx
3463; AVX512-NEXT:    popq %r14
3464; AVX512-NEXT:    popq %rbp
3465; AVX512-NEXT:    retq
3466  %1 = fptrunc <4 x double> %a0 to <4 x half>
3467  %2 = bitcast <4 x half> %1 to <4 x i16>
3468  %3 = shufflevector <4 x i16> %2, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
3469  store <8 x i16> %3, <8 x i16>* %a1
3470  ret void
3471}
3472
3473define void @store_cvt_4f64_to_8i16_zero(<4 x double> %a0, <8 x i16>* %a1) nounwind {
3474; AVX1-LABEL: store_cvt_4f64_to_8i16_zero:
3475; AVX1:       # %bb.0:
3476; AVX1-NEXT:    pushq %rbp
3477; AVX1-NEXT:    pushq %r14
3478; AVX1-NEXT:    pushq %rbx
3479; AVX1-NEXT:    subq $32, %rsp
3480; AVX1-NEXT:    movq %rdi, %r14
3481; AVX1-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3482; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3483; AVX1-NEXT:    vzeroupper
3484; AVX1-NEXT:    callq __truncdfhf2
3485; AVX1-NEXT:    movl %eax, %ebp
3486; AVX1-NEXT:    shll $16, %ebp
3487; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3488; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3489; AVX1-NEXT:    vzeroupper
3490; AVX1-NEXT:    callq __truncdfhf2
3491; AVX1-NEXT:    movzwl %ax, %ebx
3492; AVX1-NEXT:    orl %ebp, %ebx
3493; AVX1-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3494; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3495; AVX1-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3496; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3497; AVX1-NEXT:    vzeroupper
3498; AVX1-NEXT:    callq __truncdfhf2
3499; AVX1-NEXT:    movl %eax, %ebp
3500; AVX1-NEXT:    shll $16, %ebp
3501; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3502; AVX1-NEXT:    callq __truncdfhf2
3503; AVX1-NEXT:    movzwl %ax, %eax
3504; AVX1-NEXT:    orl %ebp, %eax
3505; AVX1-NEXT:    shlq $32, %rax
3506; AVX1-NEXT:    orq %rbx, %rax
3507; AVX1-NEXT:    vmovq %rax, %xmm0
3508; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
3509; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
3510; AVX1-NEXT:    vmovdqa %xmm0, (%r14)
3511; AVX1-NEXT:    addq $32, %rsp
3512; AVX1-NEXT:    popq %rbx
3513; AVX1-NEXT:    popq %r14
3514; AVX1-NEXT:    popq %rbp
3515; AVX1-NEXT:    retq
3516;
3517; AVX2-SLOW-LABEL: store_cvt_4f64_to_8i16_zero:
3518; AVX2-SLOW:       # %bb.0:
3519; AVX2-SLOW-NEXT:    pushq %rbp
3520; AVX2-SLOW-NEXT:    pushq %r14
3521; AVX2-SLOW-NEXT:    pushq %rbx
3522; AVX2-SLOW-NEXT:    subq $32, %rsp
3523; AVX2-SLOW-NEXT:    movq %rdi, %r14
3524; AVX2-SLOW-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3525; AVX2-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3526; AVX2-SLOW-NEXT:    vzeroupper
3527; AVX2-SLOW-NEXT:    callq __truncdfhf2
3528; AVX2-SLOW-NEXT:    movl %eax, %ebp
3529; AVX2-SLOW-NEXT:    shll $16, %ebp
3530; AVX2-SLOW-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3531; AVX2-SLOW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3532; AVX2-SLOW-NEXT:    vzeroupper
3533; AVX2-SLOW-NEXT:    callq __truncdfhf2
3534; AVX2-SLOW-NEXT:    movzwl %ax, %ebx
3535; AVX2-SLOW-NEXT:    orl %ebp, %ebx
3536; AVX2-SLOW-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3537; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm0
3538; AVX2-SLOW-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3539; AVX2-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3540; AVX2-SLOW-NEXT:    vzeroupper
3541; AVX2-SLOW-NEXT:    callq __truncdfhf2
3542; AVX2-SLOW-NEXT:    movl %eax, %ebp
3543; AVX2-SLOW-NEXT:    shll $16, %ebp
3544; AVX2-SLOW-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3545; AVX2-SLOW-NEXT:    callq __truncdfhf2
3546; AVX2-SLOW-NEXT:    movzwl %ax, %eax
3547; AVX2-SLOW-NEXT:    orl %ebp, %eax
3548; AVX2-SLOW-NEXT:    shlq $32, %rax
3549; AVX2-SLOW-NEXT:    orq %rbx, %rax
3550; AVX2-SLOW-NEXT:    vmovq %rax, %xmm0
3551; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
3552; AVX2-SLOW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
3553; AVX2-SLOW-NEXT:    vmovdqa %xmm0, (%r14)
3554; AVX2-SLOW-NEXT:    addq $32, %rsp
3555; AVX2-SLOW-NEXT:    popq %rbx
3556; AVX2-SLOW-NEXT:    popq %r14
3557; AVX2-SLOW-NEXT:    popq %rbp
3558; AVX2-SLOW-NEXT:    retq
3559;
3560; AVX2-FAST-LABEL: store_cvt_4f64_to_8i16_zero:
3561; AVX2-FAST:       # %bb.0:
3562; AVX2-FAST-NEXT:    pushq %rbp
3563; AVX2-FAST-NEXT:    pushq %r14
3564; AVX2-FAST-NEXT:    pushq %rbx
3565; AVX2-FAST-NEXT:    subq $32, %rsp
3566; AVX2-FAST-NEXT:    movq %rdi, %r14
3567; AVX2-FAST-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3568; AVX2-FAST-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3569; AVX2-FAST-NEXT:    vzeroupper
3570; AVX2-FAST-NEXT:    callq __truncdfhf2
3571; AVX2-FAST-NEXT:    movl %eax, %ebp
3572; AVX2-FAST-NEXT:    shll $16, %ebp
3573; AVX2-FAST-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3574; AVX2-FAST-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3575; AVX2-FAST-NEXT:    vzeroupper
3576; AVX2-FAST-NEXT:    callq __truncdfhf2
3577; AVX2-FAST-NEXT:    movzwl %ax, %ebx
3578; AVX2-FAST-NEXT:    orl %ebp, %ebx
3579; AVX2-FAST-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3580; AVX2-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm0
3581; AVX2-FAST-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3582; AVX2-FAST-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3583; AVX2-FAST-NEXT:    vzeroupper
3584; AVX2-FAST-NEXT:    callq __truncdfhf2
3585; AVX2-FAST-NEXT:    movl %eax, %ebp
3586; AVX2-FAST-NEXT:    shll $16, %ebp
3587; AVX2-FAST-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3588; AVX2-FAST-NEXT:    callq __truncdfhf2
3589; AVX2-FAST-NEXT:    movzwl %ax, %eax
3590; AVX2-FAST-NEXT:    orl %ebp, %eax
3591; AVX2-FAST-NEXT:    shlq $32, %rax
3592; AVX2-FAST-NEXT:    orq %rbx, %rax
3593; AVX2-FAST-NEXT:    vmovq %rax, %xmm0
3594; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero
3595; AVX2-FAST-NEXT:    vmovdqa %xmm0, (%r14)
3596; AVX2-FAST-NEXT:    addq $32, %rsp
3597; AVX2-FAST-NEXT:    popq %rbx
3598; AVX2-FAST-NEXT:    popq %r14
3599; AVX2-FAST-NEXT:    popq %rbp
3600; AVX2-FAST-NEXT:    retq
3601;
3602; AVX512F-LABEL: store_cvt_4f64_to_8i16_zero:
3603; AVX512F:       # %bb.0:
3604; AVX512F-NEXT:    pushq %rbp
3605; AVX512F-NEXT:    pushq %r14
3606; AVX512F-NEXT:    pushq %rbx
3607; AVX512F-NEXT:    subq $32, %rsp
3608; AVX512F-NEXT:    movq %rdi, %r14
3609; AVX512F-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3610; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3611; AVX512F-NEXT:    vzeroupper
3612; AVX512F-NEXT:    callq __truncdfhf2
3613; AVX512F-NEXT:    movl %eax, %ebp
3614; AVX512F-NEXT:    shll $16, %ebp
3615; AVX512F-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3616; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3617; AVX512F-NEXT:    vzeroupper
3618; AVX512F-NEXT:    callq __truncdfhf2
3619; AVX512F-NEXT:    movzwl %ax, %ebx
3620; AVX512F-NEXT:    orl %ebp, %ebx
3621; AVX512F-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3622; AVX512F-NEXT:    vextractf128 $1, %ymm0, %xmm0
3623; AVX512F-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3624; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3625; AVX512F-NEXT:    vzeroupper
3626; AVX512F-NEXT:    callq __truncdfhf2
3627; AVX512F-NEXT:    movl %eax, %ebp
3628; AVX512F-NEXT:    shll $16, %ebp
3629; AVX512F-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3630; AVX512F-NEXT:    callq __truncdfhf2
3631; AVX512F-NEXT:    movzwl %ax, %eax
3632; AVX512F-NEXT:    orl %ebp, %eax
3633; AVX512F-NEXT:    shlq $32, %rax
3634; AVX512F-NEXT:    orq %rbx, %rax
3635; AVX512F-NEXT:    vmovq %rax, %xmm0
3636; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
3637; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
3638; AVX512F-NEXT:    vmovdqa %xmm0, (%r14)
3639; AVX512F-NEXT:    addq $32, %rsp
3640; AVX512F-NEXT:    popq %rbx
3641; AVX512F-NEXT:    popq %r14
3642; AVX512F-NEXT:    popq %rbp
3643; AVX512F-NEXT:    retq
3644;
3645; AVX512VL-LABEL: store_cvt_4f64_to_8i16_zero:
3646; AVX512VL:       # %bb.0:
3647; AVX512VL-NEXT:    pushq %rbp
3648; AVX512VL-NEXT:    pushq %r14
3649; AVX512VL-NEXT:    pushq %rbx
3650; AVX512VL-NEXT:    subq $32, %rsp
3651; AVX512VL-NEXT:    movq %rdi, %r14
3652; AVX512VL-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
3653; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3654; AVX512VL-NEXT:    vzeroupper
3655; AVX512VL-NEXT:    callq __truncdfhf2
3656; AVX512VL-NEXT:    movl %eax, %ebp
3657; AVX512VL-NEXT:    shll $16, %ebp
3658; AVX512VL-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
3659; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3660; AVX512VL-NEXT:    vzeroupper
3661; AVX512VL-NEXT:    callq __truncdfhf2
3662; AVX512VL-NEXT:    movzwl %ax, %ebx
3663; AVX512VL-NEXT:    orl %ebp, %ebx
3664; AVX512VL-NEXT:    vmovupd (%rsp), %ymm0 # 32-byte Reload
3665; AVX512VL-NEXT:    vextractf128 $1, %ymm0, %xmm0
3666; AVX512VL-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill
3667; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3668; AVX512VL-NEXT:    vzeroupper
3669; AVX512VL-NEXT:    callq __truncdfhf2
3670; AVX512VL-NEXT:    movl %eax, %ebp
3671; AVX512VL-NEXT:    shll $16, %ebp
3672; AVX512VL-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
3673; AVX512VL-NEXT:    callq __truncdfhf2
3674; AVX512VL-NEXT:    movzwl %ax, %eax
3675; AVX512VL-NEXT:    orl %ebp, %eax
3676; AVX512VL-NEXT:    shlq $32, %rax
3677; AVX512VL-NEXT:    orq %rbx, %rax
3678; AVX512VL-NEXT:    vmovq %rax, %xmm0
3679; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero
3680; AVX512VL-NEXT:    vmovdqa %xmm0, (%r14)
3681; AVX512VL-NEXT:    addq $32, %rsp
3682; AVX512VL-NEXT:    popq %rbx
3683; AVX512VL-NEXT:    popq %r14
3684; AVX512VL-NEXT:    popq %rbp
3685; AVX512VL-NEXT:    retq
3686  %1 = fptrunc <4 x double> %a0 to <4 x half>
3687  %2 = bitcast <4 x half> %1 to <4 x i16>
3688  %3 = shufflevector <4 x i16> %2, <4 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
3689  store <8 x i16> %3, <8 x i16>* %a1
3690  ret void
3691}
3692
3693define void @store_cvt_8f64_to_8i16(<8 x double> %a0, <8 x i16>* %a1) nounwind {
3694; AVX1-LABEL: store_cvt_8f64_to_8i16:
3695; AVX1:       # %bb.0:
3696; AVX1-NEXT:    pushq %rbp
3697; AVX1-NEXT:    pushq %r15
3698; AVX1-NEXT:    pushq %r14
3699; AVX1-NEXT:    pushq %r13
3700; AVX1-NEXT:    pushq %r12
3701; AVX1-NEXT:    pushq %rbx
3702; AVX1-NEXT:    subq $136, %rsp
3703; AVX1-NEXT:    movq %rdi, %rbx
3704; AVX1-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3705; AVX1-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3706; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3707; AVX1-NEXT:    vzeroupper
3708; AVX1-NEXT:    callq __truncdfhf2
3709; AVX1-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
3710; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3711; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3712; AVX1-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3713; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3714; AVX1-NEXT:    vzeroupper
3715; AVX1-NEXT:    callq __truncdfhf2
3716; AVX1-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
3717; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
3718; AVX1-NEXT:    # xmm0 = mem[1,0]
3719; AVX1-NEXT:    callq __truncdfhf2
3720; AVX1-NEXT:    movl %eax, %r12d
3721; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3722; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
3723; AVX1-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3724; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3725; AVX1-NEXT:    vzeroupper
3726; AVX1-NEXT:    callq __truncdfhf2
3727; AVX1-NEXT:    movl %eax, %r13d
3728; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3729; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3730; AVX1-NEXT:    vzeroupper
3731; AVX1-NEXT:    callq __truncdfhf2
3732; AVX1-NEXT:    movl %eax, %ebp
3733; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3734; AVX1-NEXT:    callq __truncdfhf2
3735; AVX1-NEXT:    movl %eax, %r14d
3736; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3737; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3738; AVX1-NEXT:    vzeroupper
3739; AVX1-NEXT:    callq __truncdfhf2
3740; AVX1-NEXT:    movl %eax, %r15d
3741; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3742; AVX1-NEXT:    callq __truncdfhf2
3743; AVX1-NEXT:    movw %ax, 12(%rbx)
3744; AVX1-NEXT:    movw %r15w, 8(%rbx)
3745; AVX1-NEXT:    movw %r14w, 4(%rbx)
3746; AVX1-NEXT:    movw %bp, (%rbx)
3747; AVX1-NEXT:    movw %r13w, 14(%rbx)
3748; AVX1-NEXT:    movw %r12w, 10(%rbx)
3749; AVX1-NEXT:    movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
3750; AVX1-NEXT:    movw %ax, 6(%rbx)
3751; AVX1-NEXT:    movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
3752; AVX1-NEXT:    movw %ax, 2(%rbx)
3753; AVX1-NEXT:    addq $136, %rsp
3754; AVX1-NEXT:    popq %rbx
3755; AVX1-NEXT:    popq %r12
3756; AVX1-NEXT:    popq %r13
3757; AVX1-NEXT:    popq %r14
3758; AVX1-NEXT:    popq %r15
3759; AVX1-NEXT:    popq %rbp
3760; AVX1-NEXT:    retq
3761;
3762; AVX2-LABEL: store_cvt_8f64_to_8i16:
3763; AVX2:       # %bb.0:
3764; AVX2-NEXT:    pushq %rbp
3765; AVX2-NEXT:    pushq %r15
3766; AVX2-NEXT:    pushq %r14
3767; AVX2-NEXT:    pushq %r13
3768; AVX2-NEXT:    pushq %r12
3769; AVX2-NEXT:    pushq %rbx
3770; AVX2-NEXT:    subq $136, %rsp
3771; AVX2-NEXT:    movq %rdi, %rbx
3772; AVX2-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3773; AVX2-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3774; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3775; AVX2-NEXT:    vzeroupper
3776; AVX2-NEXT:    callq __truncdfhf2
3777; AVX2-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
3778; AVX2-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3779; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
3780; AVX2-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3781; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3782; AVX2-NEXT:    vzeroupper
3783; AVX2-NEXT:    callq __truncdfhf2
3784; AVX2-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
3785; AVX2-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
3786; AVX2-NEXT:    # xmm0 = mem[1,0]
3787; AVX2-NEXT:    callq __truncdfhf2
3788; AVX2-NEXT:    movl %eax, %r12d
3789; AVX2-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3790; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
3791; AVX2-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3792; AVX2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3793; AVX2-NEXT:    vzeroupper
3794; AVX2-NEXT:    callq __truncdfhf2
3795; AVX2-NEXT:    movl %eax, %r13d
3796; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3797; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3798; AVX2-NEXT:    vzeroupper
3799; AVX2-NEXT:    callq __truncdfhf2
3800; AVX2-NEXT:    movl %eax, %ebp
3801; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3802; AVX2-NEXT:    callq __truncdfhf2
3803; AVX2-NEXT:    movl %eax, %r14d
3804; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3805; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3806; AVX2-NEXT:    vzeroupper
3807; AVX2-NEXT:    callq __truncdfhf2
3808; AVX2-NEXT:    movl %eax, %r15d
3809; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3810; AVX2-NEXT:    callq __truncdfhf2
3811; AVX2-NEXT:    movw %ax, 12(%rbx)
3812; AVX2-NEXT:    movw %r15w, 8(%rbx)
3813; AVX2-NEXT:    movw %r14w, 4(%rbx)
3814; AVX2-NEXT:    movw %bp, (%rbx)
3815; AVX2-NEXT:    movw %r13w, 14(%rbx)
3816; AVX2-NEXT:    movw %r12w, 10(%rbx)
3817; AVX2-NEXT:    movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
3818; AVX2-NEXT:    movw %ax, 6(%rbx)
3819; AVX2-NEXT:    movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
3820; AVX2-NEXT:    movw %ax, 2(%rbx)
3821; AVX2-NEXT:    addq $136, %rsp
3822; AVX2-NEXT:    popq %rbx
3823; AVX2-NEXT:    popq %r12
3824; AVX2-NEXT:    popq %r13
3825; AVX2-NEXT:    popq %r14
3826; AVX2-NEXT:    popq %r15
3827; AVX2-NEXT:    popq %rbp
3828; AVX2-NEXT:    retq
3829;
3830; AVX512-LABEL: store_cvt_8f64_to_8i16:
3831; AVX512:       # %bb.0:
3832; AVX512-NEXT:    pushq %rbp
3833; AVX512-NEXT:    pushq %r15
3834; AVX512-NEXT:    pushq %r14
3835; AVX512-NEXT:    pushq %r13
3836; AVX512-NEXT:    pushq %r12
3837; AVX512-NEXT:    pushq %rbx
3838; AVX512-NEXT:    subq $200, %rsp
3839; AVX512-NEXT:    movq %rdi, %rbx
3840; AVX512-NEXT:    vmovupd %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
3841; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3842; AVX512-NEXT:    vzeroupper
3843; AVX512-NEXT:    callq __truncdfhf2
3844; AVX512-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
3845; AVX512-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
3846; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
3847; AVX512-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3848; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3849; AVX512-NEXT:    vzeroupper
3850; AVX512-NEXT:    callq __truncdfhf2
3851; AVX512-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
3852; AVX512-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
3853; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
3854; AVX512-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3855; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3856; AVX512-NEXT:    vzeroupper
3857; AVX512-NEXT:    callq __truncdfhf2
3858; AVX512-NEXT:    movl %eax, %r12d
3859; AVX512-NEXT:    vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3860; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
3861; AVX512-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3862; AVX512-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
3863; AVX512-NEXT:    vzeroupper
3864; AVX512-NEXT:    callq __truncdfhf2
3865; AVX512-NEXT:    movl %eax, %r13d
3866; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
3867; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
3868; AVX512-NEXT:    vzeroupper
3869; AVX512-NEXT:    callq __truncdfhf2
3870; AVX512-NEXT:    movl %eax, %ebp
3871; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3872; AVX512-NEXT:    callq __truncdfhf2
3873; AVX512-NEXT:    movl %eax, %r14d
3874; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3875; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
3876; AVX512-NEXT:    vzeroupper
3877; AVX512-NEXT:    callq __truncdfhf2
3878; AVX512-NEXT:    movl %eax, %r15d
3879; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3880; AVX512-NEXT:    callq __truncdfhf2
3881; AVX512-NEXT:    movw %ax, 12(%rbx)
3882; AVX512-NEXT:    movw %r15w, 8(%rbx)
3883; AVX512-NEXT:    movw %r14w, 4(%rbx)
3884; AVX512-NEXT:    movw %bp, (%rbx)
3885; AVX512-NEXT:    movw %r13w, 14(%rbx)
3886; AVX512-NEXT:    movw %r12w, 10(%rbx)
3887; AVX512-NEXT:    movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
3888; AVX512-NEXT:    movw %ax, 6(%rbx)
3889; AVX512-NEXT:    movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
3890; AVX512-NEXT:    movw %ax, 2(%rbx)
3891; AVX512-NEXT:    addq $200, %rsp
3892; AVX512-NEXT:    popq %rbx
3893; AVX512-NEXT:    popq %r12
3894; AVX512-NEXT:    popq %r13
3895; AVX512-NEXT:    popq %r14
3896; AVX512-NEXT:    popq %r15
3897; AVX512-NEXT:    popq %rbp
3898; AVX512-NEXT:    retq
3899  %1 = fptrunc <8 x double> %a0 to <8 x half>
3900  %2 = bitcast <8 x half> %1 to <8 x i16>
3901  store <8 x i16> %2, <8 x i16>* %a1
3902  ret void
3903}
3904