1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
3; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+sse4.2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE42
4; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
5; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
6
7define void @v3i64(<2 x i64> %a, <2 x i64> %b, <3 x i64>* %p) nounwind {
8; SSE2-LABEL: v3i64:
9; SSE2:       # BB#0:
10; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
11; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
12; SSE2-NEXT:    movq %xmm2, 16(%rdi)
13; SSE2-NEXT:    movdqa %xmm0, (%rdi)
14; SSE2-NEXT:    retq
15;
16; SSE42-LABEL: v3i64:
17; SSE42:       # BB#0:
18; SSE42-NEXT:    pextrq $1, %xmm0, 16(%rdi)
19; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
20; SSE42-NEXT:    movdqa %xmm0, (%rdi)
21; SSE42-NEXT:    retq
22;
23; AVX1-LABEL: v3i64:
24; AVX1:       # BB#0:
25; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0]
26; AVX1-NEXT:    vpextrq $1, %xmm0, 16(%rdi)
27; AVX1-NEXT:    vmovapd %xmm1, (%rdi)
28; AVX1-NEXT:    retq
29;
30; AVX2-LABEL: v3i64:
31; AVX2:       # BB#0:
32; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<def>
33; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
34; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
35; AVX2-NEXT:    vpextrq $1, %xmm0, 16(%rdi)
36; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
37; AVX2-NEXT:    vzeroupper
38; AVX2-NEXT:    retq
39  %r = shufflevector <2 x i64> %a, <2 x i64> %b, <3 x i32> <i32 0, i32 2, i32 1>
40  store <3 x i64> %r, <3 x i64>* %p
41  ret void
42}
43define void @v3f64(<2 x double> %a, <2 x double> %b, <3 x double>* %p) nounwind {
44; SSE-LABEL: v3f64:
45; SSE:       # BB#0:
46; SSE-NEXT:    movhpd %xmm0, 16(%rdi)
47; SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
48; SSE-NEXT:    movapd %xmm0, (%rdi)
49; SSE-NEXT:    retq
50;
51; AVX1-LABEL: v3f64:
52; AVX1:       # BB#0:
53; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0]
54; AVX1-NEXT:    vmovhpd %xmm0, 16(%rdi)
55; AVX1-NEXT:    vmovapd %xmm1, (%rdi)
56; AVX1-NEXT:    retq
57;
58; AVX2-LABEL: v3f64:
59; AVX2:       # BB#0:
60; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<def>
61; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
62; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
63; AVX2-NEXT:    vmovhpd %xmm0, 16(%rdi)
64; AVX2-NEXT:    vmovapd %xmm1, (%rdi)
65; AVX2-NEXT:    vzeroupper
66; AVX2-NEXT:    retq
67  %r = shufflevector <2 x double> %a, <2 x double> %b, <3 x i32> <i32 0, i32 2, i32 1>
68  store <3 x double> %r, <3 x double>* %p
69  ret void
70}
71
72define void @v3i32(<2 x i32> %a, <2 x i32> %b, <3 x i32>* %p) nounwind {
73; SSE2-LABEL: v3i32:
74; SSE2:       # BB#0:
75; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
76; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
77; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
78; SSE2-NEXT:    movd %xmm0, 8(%rdi)
79; SSE2-NEXT:    movq %xmm2, (%rdi)
80; SSE2-NEXT:    retq
81;
82; SSE42-LABEL: v3i32:
83; SSE42:       # BB#0:
84; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
85; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
86; SSE42-NEXT:    pextrd $2, %xmm0, 8(%rdi)
87; SSE42-NEXT:    movq %xmm1, (%rdi)
88; SSE42-NEXT:    retq
89;
90; AVX1-LABEL: v3i32:
91; AVX1:       # BB#0:
92; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
93; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
94; AVX1-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
95; AVX1-NEXT:    vmovq %xmm1, (%rdi)
96; AVX1-NEXT:    retq
97;
98; AVX2-LABEL: v3i32:
99; AVX2:       # BB#0:
100; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
101; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2,3]
102; AVX2-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
103; AVX2-NEXT:    vmovq %xmm1, (%rdi)
104; AVX2-NEXT:    retq
105  %r = shufflevector <2 x i32> %a, <2 x i32> %b, <3 x i32> <i32 0, i32 2, i32 1>
106  store <3 x i32> %r, <3 x i32>* %p
107  ret void
108}
109
110define void @v5i16(<4 x i16> %a, <4 x i16> %b, <5 x i16>* %p) nounwind {
111; SSE2-LABEL: v5i16:
112; SSE2:       # BB#0:
113; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
114; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
115; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
116; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
117; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
118; SSE2-NEXT:    pextrw $6, %xmm0, %eax
119; SSE2-NEXT:    movw %ax, 8(%rdi)
120; SSE2-NEXT:    movq %xmm2, (%rdi)
121; SSE2-NEXT:    retq
122;
123; SSE42-LABEL: v5i16:
124; SSE42:       # BB#0:
125; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
126; SSE42-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
127; SSE42-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
128; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
129; SSE42-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
130; SSE42-NEXT:    pextrw $6, %xmm0, 8(%rdi)
131; SSE42-NEXT:    movq %xmm2, (%rdi)
132; SSE42-NEXT:    retq
133;
134; AVX-LABEL: v5i16:
135; AVX:       # BB#0:
136; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
137; AVX-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
138; AVX-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
139; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
140; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
141; AVX-NEXT:    vpextrw $6, %xmm0, 8(%rdi)
142; AVX-NEXT:    vmovq %xmm1, (%rdi)
143; AVX-NEXT:    retq
144  %r = shufflevector <4 x i16> %a, <4 x i16> %b, <5 x i32> <i32 0, i32 5, i32 1, i32 6, i32 3>
145  store <5 x i16> %r, <5 x i16>* %p
146  ret void
147}
148
149define void @v5i32(<4 x i32> %a, <4 x i32> %b, <5 x i32>* %p) nounwind {
150; SSE2-LABEL: v5i32:
151; SSE2:       # BB#0:
152; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,2,2,3]
153; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
154; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
155; SSE2-NEXT:    movd %xmm2, 16(%rdi)
156; SSE2-NEXT:    movdqa %xmm0, (%rdi)
157; SSE2-NEXT:    retq
158;
159; SSE42-LABEL: v5i32:
160; SSE42:       # BB#0:
161; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
162; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
163; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
164; SSE42-NEXT:    pextrd $3, %xmm0, 16(%rdi)
165; SSE42-NEXT:    movdqa %xmm2, (%rdi)
166; SSE42-NEXT:    retq
167;
168; AVX1-LABEL: v5i32:
169; AVX1:       # BB#0:
170; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[0,1],xmm1[1,2]
171; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,1,3]
172; AVX1-NEXT:    vpextrd $3, %xmm0, 16(%rdi)
173; AVX1-NEXT:    vmovaps %xmm1, (%rdi)
174; AVX1-NEXT:    retq
175;
176; AVX2-LABEL: v5i32:
177; AVX2:       # BB#0:
178; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<def>
179; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
180; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = <0,5,1,6,3,u,u,u>
181; AVX2-NEXT:    vpermd %ymm1, %ymm2, %ymm1
182; AVX2-NEXT:    vpextrd $3, %xmm0, 16(%rdi)
183; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
184; AVX2-NEXT:    vzeroupper
185; AVX2-NEXT:    retq
186  %r = shufflevector <4 x i32> %a, <4 x i32> %b, <5 x i32> <i32 0, i32 5, i32 1, i32 6, i32 3>
187  store <5 x i32> %r, <5 x i32>* %p
188  ret void
189}
190
191define void @v5f32(<4 x float> %a, <4 x float> %b, <5 x float>* %p) nounwind {
192; SSE2-LABEL: v5f32:
193; SSE2:       # BB#0:
194; SSE2-NEXT:    movaps %xmm0, %xmm2
195; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[1,2]
196; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2,1,3]
197; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
198; SSE2-NEXT:    movss %xmm0, 16(%rdi)
199; SSE2-NEXT:    movaps %xmm2, (%rdi)
200; SSE2-NEXT:    retq
201;
202; SSE42-LABEL: v5f32:
203; SSE42:       # BB#0:
204; SSE42-NEXT:    extractps $3, %xmm0, 16(%rdi)
205; SSE42-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[1,2]
206; SSE42-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
207; SSE42-NEXT:    movaps %xmm0, (%rdi)
208; SSE42-NEXT:    retq
209;
210; AVX1-LABEL: v5f32:
211; AVX1:       # BB#0:
212; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[0,1],xmm1[1,2]
213; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,1,3]
214; AVX1-NEXT:    vextractps $3, %xmm0, 16(%rdi)
215; AVX1-NEXT:    vmovaps %xmm1, (%rdi)
216; AVX1-NEXT:    retq
217;
218; AVX2-LABEL: v5f32:
219; AVX2:       # BB#0:
220; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<def>
221; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
222; AVX2-NEXT:    vmovaps {{.*#+}} ymm2 = <0,5,1,6,3,u,u,u>
223; AVX2-NEXT:    vpermps %ymm1, %ymm2, %ymm1
224; AVX2-NEXT:    vextractps $3, %xmm0, 16(%rdi)
225; AVX2-NEXT:    vmovaps %xmm1, (%rdi)
226; AVX2-NEXT:    vzeroupper
227; AVX2-NEXT:    retq
228  %r = shufflevector <4 x float> %a, <4 x float> %b, <5 x i32> <i32 0, i32 5, i32 1, i32 6, i32 3>
229  store <5 x float> %r, <5 x float>* %p
230  ret void
231}
232
233define void @v7i8(<4 x i8> %a, <4 x i8> %b, <7 x i8>* %p) nounwind {
234; SSE2-LABEL: v7i8:
235; SSE2:       # BB#0:
236; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
237; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
238; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,1,3,4,5,6,7]
239; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,255,0,255,0,255,255,255,255,255,255,255,255,255,255,255]
240; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,6,6,7]
241; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0]
242; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,5,7]
243; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
244; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
245; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
246; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,0,2,1,4,5,6,7]
247; SSE2-NEXT:    pand %xmm2, %xmm3
248; SSE2-NEXT:    pandn %xmm0, %xmm2
249; SSE2-NEXT:    por %xmm3, %xmm2
250; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
251; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
252; SSE2-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp)
253; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
254; SSE2-NEXT:    movb %al, 6(%rdi)
255; SSE2-NEXT:    movd %xmm0, (%rdi)
256; SSE2-NEXT:    pextrw $4, %xmm2, %eax
257; SSE2-NEXT:    movw %ax, 4(%rdi)
258; SSE2-NEXT:    retq
259;
260; SSE42-LABEL: v7i8:
261; SSE42:       # BB#0:
262; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[12],zero,xmm0[4],zero,zero,xmm0[u,u,u,u,u,u,u,u,u]
263; SSE42-NEXT:    pextrb $0, %xmm1, 6(%rdi)
264; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = zero,xmm1[8],zero,xmm1[8],zero,xmm1[12,0,u,u,u,u,u,u,u,u,u]
265; SSE42-NEXT:    por %xmm0, %xmm1
266; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
267; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
268; SSE42-NEXT:    pextrw $4, %xmm1, 4(%rdi)
269; SSE42-NEXT:    movd %xmm0, (%rdi)
270; SSE42-NEXT:    retq
271;
272; AVX-LABEL: v7i8:
273; AVX:       # BB#0:
274; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[12],zero,xmm0[4],zero,zero,xmm0[u,u,u,u,u,u,u,u,u]
275; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm1[8],zero,xmm1[8],zero,xmm1[12,0,u,u,u,u,u,u,u,u,u]
276; AVX-NEXT:    vpor %xmm0, %xmm2, %xmm0
277; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
278; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
279; AVX-NEXT:    vpextrb $0, %xmm1, 6(%rdi)
280; AVX-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
281; AVX-NEXT:    vmovd %xmm2, (%rdi)
282; AVX-NEXT:    retq
283  %r = shufflevector <4 x i8> %a, <4 x i8> %b, <7 x i32> <i32 0, i32 6, i32 3, i32 6, i32 1, i32 7, i32 4>
284  store <7 x i8> %r, <7 x i8>* %p
285  ret void
286}
287
288define void @v7i16(<4 x i16> %a, <4 x i16> %b, <7 x i16>* %p) nounwind {
289; SSE2-LABEL: v7i16:
290; SSE2:       # BB#0:
291; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
292; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,65535,0,65535,0,65535,65535,65535]
293; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,1,0,3]
294; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,0,2,0,4,5,6,7]
295; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,4,7]
296; SSE2-NEXT:    pand %xmm2, %xmm3
297; SSE2-NEXT:    pandn %xmm0, %xmm2
298; SSE2-NEXT:    por %xmm3, %xmm2
299; SSE2-NEXT:    movd %xmm1, %eax
300; SSE2-NEXT:    movw %ax, 12(%rdi)
301; SSE2-NEXT:    movq %xmm2, (%rdi)
302; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
303; SSE2-NEXT:    movd %xmm0, 8(%rdi)
304; SSE2-NEXT:    retq
305;
306; SSE42-LABEL: v7i16:
307; SSE42:       # BB#0:
308; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
309; SSE42-NEXT:    pextrw $0, %xmm1, 12(%rdi)
310; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[8,9,8,9,4,5,8,9,0,1,12,13,0,1,14,15]
311; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5,6,7]
312; SSE42-NEXT:    pextrd $2, %xmm1, 8(%rdi)
313; SSE42-NEXT:    movq %xmm1, (%rdi)
314; SSE42-NEXT:    retq
315;
316; AVX-LABEL: v7i16:
317; AVX:       # BB#0:
318; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
319; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = xmm1[8,9,8,9,4,5,8,9,0,1,12,13,0,1,14,15]
320; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5,6,7]
321; AVX-NEXT:    vpextrw $0, %xmm1, 12(%rdi)
322; AVX-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
323; AVX-NEXT:    vmovq %xmm0, (%rdi)
324; AVX-NEXT:    retq
325  %r = shufflevector <4 x i16> %a, <4 x i16> %b, <7 x i32> <i32 0, i32 6, i32 3, i32 6, i32 1, i32 7, i32 4>
326  store <7 x i16> %r, <7 x i16>* %p
327  ret void
328}
329
330
331define void @v7i32(<4 x i32> %a, <4 x i32> %b, <7 x i32>* %p) nounwind {
332; SSE2-LABEL: v7i32:
333; SSE2:       # BB#0:
334; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,1,2,2]
335; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,1,0,3]
336; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
337; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[3,0]
338; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3]
339; SSE2-NEXT:    movd %xmm1, 24(%rdi)
340; SSE2-NEXT:    movlps %xmm0, 16(%rdi)
341; SSE2-NEXT:    movdqa %xmm3, (%rdi)
342; SSE2-NEXT:    retq
343;
344; SSE42-LABEL: v7i32:
345; SSE42:       # BB#0:
346; SSE42-NEXT:    movdqa %xmm1, %xmm2
347; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3],xmm2[4,5,6,7]
348; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5],xmm0[6,7]
349; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,3,2]
350; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,0,3]
351; SSE42-NEXT:    movd %xmm1, 24(%rdi)
352; SSE42-NEXT:    movq %xmm2, 16(%rdi)
353; SSE42-NEXT:    movdqa %xmm0, (%rdi)
354; SSE42-NEXT:    retq
355;
356; AVX1-LABEL: v7i32:
357; AVX1:       # BB#0:
358; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm0[0,1],xmm1[2],xmm0[3]
359; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[0,2,3,2]
360; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3]
361; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,3,0,3]
362; AVX1-NEXT:    vmovd %xmm1, 24(%rdi)
363; AVX1-NEXT:    vmovlps %xmm0, 16(%rdi)
364; AVX1-NEXT:    vmovaps %xmm2, (%rdi)
365; AVX1-NEXT:    retq
366;
367; AVX2-LABEL: v7i32:
368; AVX2:       # BB#0:
369; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<def>
370; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
371; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = <0,6,3,6,1,7,4,u>
372; AVX2-NEXT:    vpermd %ymm0, %ymm2, %ymm0
373; AVX2-NEXT:    vmovd %xmm1, 24(%rdi)
374; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
375; AVX2-NEXT:    vmovq %xmm1, 16(%rdi)
376; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
377; AVX2-NEXT:    vzeroupper
378; AVX2-NEXT:    retq
379  %r = shufflevector <4 x i32> %a, <4 x i32> %b, <7 x i32> <i32 0, i32 6, i32 3, i32 6, i32 1, i32 7, i32 4>
380  store <7 x i32> %r, <7 x i32>* %p
381  ret void
382}
383
384define void @v12i8(<8 x i8> %a, <8 x i8> %b, <12 x i8>* %p) nounwind {
385; SSE2-LABEL: v12i8:
386; SSE2:       # BB#0:
387; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
388; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
389; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
390; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,255,255]
391; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
392; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
393; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
394; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
395; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
396; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,1,4,5,6,7]
397; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,4]
398; SSE2-NEXT:    packuswb %xmm3, %xmm0
399; SSE2-NEXT:    pand %xmm2, %xmm0
400; SSE2-NEXT:    pandn %xmm1, %xmm2
401; SSE2-NEXT:    por %xmm0, %xmm2
402; SSE2-NEXT:    movq %xmm2, (%rdi)
403; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
404; SSE2-NEXT:    movd %xmm0, 8(%rdi)
405; SSE2-NEXT:    retq
406;
407; SSE42-LABEL: v12i8:
408; SSE42:       # BB#0:
409; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = zero,zero,xmm1[0],zero,zero,xmm1[2],zero,zero,xmm1[4],zero,zero,xmm1[6,u,u,u,u]
410; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8],zero,xmm0[2,10],zero,xmm0[4,12],zero,xmm0[6,14],zero,xmm0[u,u,u,u]
411; SSE42-NEXT:    por %xmm1, %xmm0
412; SSE42-NEXT:    pextrd $2, %xmm0, 8(%rdi)
413; SSE42-NEXT:    movq %xmm0, (%rdi)
414; SSE42-NEXT:    retq
415;
416; AVX-LABEL: v12i8:
417; AVX:       # BB#0:
418; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,xmm1[0],zero,zero,xmm1[2],zero,zero,xmm1[4],zero,zero,xmm1[6,u,u,u,u]
419; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8],zero,xmm0[2,10],zero,xmm0[4,12],zero,xmm0[6,14],zero,xmm0[u,u,u,u]
420; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
421; AVX-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
422; AVX-NEXT:    vmovq %xmm0, (%rdi)
423; AVX-NEXT:    retq
424  %r = shufflevector <8 x i8> %a, <8 x i8> %b, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
425  store <12 x i8> %r, <12 x i8>* %p
426  ret void
427}
428
429define void @v12i16(<8 x i16> %a, <8 x i16> %b, <12 x i16>* %p) nounwind {
430; SSE2-LABEL: v12i16:
431; SSE2:       # BB#0:
432; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,0,3]
433; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [65535,65535,0,65535,65535,0,65535,65535]
434; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,6,5,4,7]
435; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,3,2,1]
436; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,2,2,1,4,5,6,7]
437; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,6,4]
438; SSE2-NEXT:    pand %xmm3, %xmm4
439; SSE2-NEXT:    pandn %xmm2, %xmm3
440; SSE2-NEXT:    por %xmm4, %xmm3
441; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
442; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,0,65535,65535,65535,65535,65535]
443; SSE2-NEXT:    pand %xmm2, %xmm1
444; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
445; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,3,1,3,4,5,6,7]
446; SSE2-NEXT:    pandn %xmm0, %xmm2
447; SSE2-NEXT:    por %xmm1, %xmm2
448; SSE2-NEXT:    movq %xmm2, 16(%rdi)
449; SSE2-NEXT:    movdqa %xmm3, (%rdi)
450; SSE2-NEXT:    retq
451;
452; SSE42-LABEL: v12i16:
453; SSE42:       # BB#0:
454; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
455; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
456; SSE42-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
457; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0],xmm3[1,2],xmm2[3,4,5,6,7]
458; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,3]
459; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,2,3,10,11,10,11,4,5,12,13]
460; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
461; SSE42-NEXT:    movdqa %xmm0, (%rdi)
462; SSE42-NEXT:    movq %xmm3, 16(%rdi)
463; SSE42-NEXT:    retq
464;
465; AVX1-LABEL: v12i16:
466; AVX1:       # BB#0:
467; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
468; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
469; AVX1-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
470; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2],xmm2[3],xmm3[4,5,6,7]
471; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,3]
472; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,2,3,10,11,10,11,4,5,12,13]
473; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
474; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
475; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
476; AVX1-NEXT:    retq
477;
478; AVX2-LABEL: v12i16:
479; AVX2:       # BB#0:
480; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
481; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
482; AVX2-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
483; AVX2-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2],xmm2[3],xmm3[4,5,6,7]
484; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
485; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,2,3,10,11,10,11,4,5,12,13]
486; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
487; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
488; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
489; AVX2-NEXT:    retq
490  %r = shufflevector <8 x i16> %a, <8 x i16> %b, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
491  store <12 x i16> %r, <12 x i16>* %p
492  ret void
493}
494
495define void @v12i32(<8 x i32> %a, <8 x i32> %b, <12 x i32>* %p) nounwind {
496; SSE2-LABEL: v12i32:
497; SSE2:       # BB#0:
498; SSE2-NEXT:    movdqa %xmm0, %xmm3
499; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
500; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,2,2]
501; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,1,0,1]
502; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm3[3,0]
503; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0,2]
504; SSE2-NEXT:    movdqa %xmm2, %xmm4
505; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,0],xmm1[1,0]
506; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm1[2,2]
507; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
508; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm4[3,0]
509; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm0[0,2]
510; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,3,2,2]
511; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
512; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0]
513; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,2]
514; SSE2-NEXT:    movaps %xmm0, 32(%rdi)
515; SSE2-NEXT:    movaps %xmm4, 16(%rdi)
516; SSE2-NEXT:    movaps %xmm3, (%rdi)
517; SSE2-NEXT:    retq
518;
519; SSE42-LABEL: v12i32:
520; SSE42:       # BB#0:
521; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,1,1]
522; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,1,0,1]
523; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5,6,7]
524; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]
525; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5],xmm4[6,7]
526; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,2,2]
527; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3],xmm4[4,5,6,7]
528; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm0[4,5],xmm4[6,7]
529; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
530; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
531; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3],xmm2[4,5,6,7]
532; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,2,3,3]
533; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7]
534; SSE42-NEXT:    movdqa %xmm0, 32(%rdi)
535; SSE42-NEXT:    movdqa %xmm4, 16(%rdi)
536; SSE42-NEXT:    movdqa %xmm3, (%rdi)
537; SSE42-NEXT:    retq
538;
539; AVX1-LABEL: v12i32:
540; AVX1:       # BB#0:
541; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,0,1]
542; AVX1-NEXT:    vmovsldup {{.*#+}} ymm2 = ymm2[0,0,2,2,4,4,6,6]
543; AVX1-NEXT:    vpermilps {{.*#+}} ymm3 = ymm0[0,u,u,1,5,u,u,6]
544; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3,4,5],ymm2[6],ymm3[7]
545; AVX1-NEXT:    vmovddup {{.*#+}} xmm3 = xmm1[0,0]
546; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm3
547; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
548; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
549; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,3],xmm3[3,3]
550; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,1]
551; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
552; AVX1-NEXT:    vmovaps %xmm0, 32(%rdi)
553; AVX1-NEXT:    vmovaps %ymm2, (%rdi)
554; AVX1-NEXT:    vzeroupper
555; AVX1-NEXT:    retq
556;
557; AVX2-LABEL: v12i32:
558; AVX2:       # BB#0:
559; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
560; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[3,3,2,3,7,7,6,7]
561; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
562; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1,2],xmm2[3]
563; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = <0,4,u,1,5,u,2,6>
564; AVX2-NEXT:    vpermd %ymm0, %ymm3, %ymm0
565; AVX2-NEXT:    vpbroadcastq %xmm1, %ymm1
566; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
567; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
568; AVX2-NEXT:    vmovdqa %xmm2, 32(%rdi)
569; AVX2-NEXT:    vzeroupper
570; AVX2-NEXT:    retq
571  %r = shufflevector <8 x i32> %a, <8 x i32> %b, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
572  store <12 x i32> %r, <12 x i32>* %p
573  ret void
574}
575
576define void @pr29025(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <12 x i8> *%p) nounwind {
577; SSE2-LABEL: pr29025:
578; SSE2:       # BB#0:
579; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255]
580; SSE2-NEXT:    pand %xmm3, %xmm1
581; SSE2-NEXT:    pand %xmm3, %xmm0
582; SSE2-NEXT:    packuswb %xmm1, %xmm0
583; SSE2-NEXT:    packuswb %xmm0, %xmm0
584; SSE2-NEXT:    pxor %xmm1, %xmm1
585; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
586; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
587; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,3,1,3,4,5,6,7]
588; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
589; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
590; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,1,4,5,6,7]
591; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,4]
592; SSE2-NEXT:    packuswb %xmm1, %xmm0
593; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,255,255]
594; SSE2-NEXT:    pand %xmm1, %xmm0
595; SSE2-NEXT:    pand %xmm3, %xmm2
596; SSE2-NEXT:    packuswb %xmm2, %xmm2
597; SSE2-NEXT:    packuswb %xmm2, %xmm2
598; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,1,1,4,5,6,7]
599; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
600; SSE2-NEXT:    pandn %xmm2, %xmm1
601; SSE2-NEXT:    por %xmm0, %xmm1
602; SSE2-NEXT:    movq %xmm1, (%rdi)
603; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
604; SSE2-NEXT:    movd %xmm0, 8(%rdi)
605; SSE2-NEXT:    retq
606;
607; SSE42-LABEL: pr29025:
608; SSE42:       # BB#0:
609; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
610; SSE42-NEXT:    pshufb %xmm3, %xmm1
611; SSE42-NEXT:    pshufb %xmm3, %xmm0
612; SSE42-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
613; SSE42-NEXT:    pshufb %xmm3, %xmm2
614; SSE42-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
615; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8,1,2,10,3,4,12,5,6,14,7,u,u,u,u]
616; SSE42-NEXT:    pextrd $2, %xmm0, 8(%rdi)
617; SSE42-NEXT:    movq %xmm0, (%rdi)
618; SSE42-NEXT:    retq
619;
620; AVX-LABEL: pr29025:
621; AVX:       # BB#0:
622; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
623; AVX-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
624; AVX-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
625; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
626; AVX-NEXT:    vpshufb %xmm3, %xmm2, %xmm1
627; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
628; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,1,2,10,3,4,12,5,6,14,7,u,u,u,u]
629; AVX-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
630; AVX-NEXT:    vmovq %xmm0, (%rdi)
631; AVX-NEXT:    retq
632  %s1 = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
633  %s2 = shufflevector <4 x i8> %c, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
634  %r = shufflevector <8 x i8> %s1, <8 x i8> %s2, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
635  store <12 x i8> %r, <12 x i8>* %p, align 1
636  ret void
637}
638
639define void @interleave_24i8_out(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8 x i8>* %q3) nounwind {
640; SSE2-LABEL: interleave_24i8_out:
641; SSE2:       # BB#0:
642; SSE2-NEXT:    movdqu (%rdi), %xmm0
643; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
644; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,0,255,255,0,255,255,255,255,255,255,255,255,255,255]
645; SSE2-NEXT:    movdqa %xmm0, %xmm2
646; SSE2-NEXT:    pand %xmm3, %xmm2
647; SSE2-NEXT:    pandn %xmm1, %xmm3
648; SSE2-NEXT:    por %xmm2, %xmm3
649; SSE2-NEXT:    pxor %xmm2, %xmm2
650; SSE2-NEXT:    movdqa %xmm3, %xmm4
651; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
652; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [65535,0,65535,65535,0,65535,65535,0]
653; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
654; SSE2-NEXT:    pand %xmm5, %xmm3
655; SSE2-NEXT:    pandn %xmm4, %xmm5
656; SSE2-NEXT:    por %xmm3, %xmm5
657; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,1,3]
658; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]
659; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,3,2,1]
660; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,3,2,1,4,5,6,7]
661; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]
662; SSE2-NEXT:    packuswb %xmm0, %xmm3
663; SSE2-NEXT:    movq %xmm3, (%rsi)
664; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [0,255,255,0,255,255,0,255,255,255,255,255,255,255,255,255]
665; SSE2-NEXT:    movdqa %xmm0, %xmm4
666; SSE2-NEXT:    pand %xmm3, %xmm4
667; SSE2-NEXT:    pandn %xmm1, %xmm3
668; SSE2-NEXT:    por %xmm4, %xmm3
669; SSE2-NEXT:    movdqa %xmm3, %xmm4
670; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
671; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [65535,65535,0,65535,65535,0,65535,65535]
672; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
673; SSE2-NEXT:    pand %xmm5, %xmm3
674; SSE2-NEXT:    pandn %xmm4, %xmm5
675; SSE2-NEXT:    por %xmm3, %xmm5
676; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[2,1,0,3,4,5,6,7]
677; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]
678; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,3,2,1]
679; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[1,2,3,0,4,5,6,7]
680; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
681; SSE2-NEXT:    packuswb %xmm0, %xmm3
682; SSE2-NEXT:    movq %xmm3, (%rdx)
683; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,0,255,255,0,255,255,0,255,255,255,255,255,255,255,255]
684; SSE2-NEXT:    pand %xmm3, %xmm0
685; SSE2-NEXT:    pandn %xmm1, %xmm3
686; SSE2-NEXT:    por %xmm0, %xmm3
687; SSE2-NEXT:    movdqa %xmm3, %xmm0
688; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
689; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,0,65535,65535,0,65535]
690; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
691; SSE2-NEXT:    pand %xmm1, %xmm3
692; SSE2-NEXT:    pandn %xmm0, %xmm1
693; SSE2-NEXT:    por %xmm3, %xmm1
694; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,0]
695; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]
696; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0]
697; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]
698; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
699; SSE2-NEXT:    packuswb %xmm0, %xmm0
700; SSE2-NEXT:    movq %xmm0, (%rcx)
701; SSE2-NEXT:    retq
702;
703; SSE42-LABEL: interleave_24i8_out:
704; SSE42:       # BB#0:
705; SSE42-NEXT:    movdqu (%rdi), %xmm0
706; SSE42-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
707; SSE42-NEXT:    movdqa %xmm1, %xmm2
708; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm2[2,5,u,u,u,u,u,u,u,u]
709; SSE42-NEXT:    movdqa %xmm0, %xmm3
710; SSE42-NEXT:    pshufb {{.*#+}} xmm3 = xmm3[0,3,6,9,12,15],zero,zero,xmm3[u,u,u,u,u,u,u,u]
711; SSE42-NEXT:    por %xmm2, %xmm3
712; SSE42-NEXT:    movq %xmm3, (%rsi)
713; SSE42-NEXT:    movdqa %xmm1, %xmm2
714; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm2[0,3,6,u,u,u,u,u,u,u,u]
715; SSE42-NEXT:    movdqa %xmm0, %xmm3
716; SSE42-NEXT:    pshufb {{.*#+}} xmm3 = xmm3[1,4,7,10,13],zero,zero,zero,xmm3[u,u,u,u,u,u,u,u]
717; SSE42-NEXT:    por %xmm2, %xmm3
718; SSE42-NEXT:    movq %xmm3, (%rdx)
719; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]
720; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
721; SSE42-NEXT:    por %xmm1, %xmm0
722; SSE42-NEXT:    movq %xmm0, (%rcx)
723; SSE42-NEXT:    retq
724;
725; AVX-LABEL: interleave_24i8_out:
726; AVX:       # BB#0:
727; AVX-NEXT:    vmovdqu (%rdi), %xmm0
728; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
729; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]
730; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]
731; AVX-NEXT:    vpor %xmm2, %xmm3, %xmm2
732; AVX-NEXT:    vmovq %xmm2, (%rsi)
733; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]
734; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
735; AVX-NEXT:    vpor %xmm2, %xmm3, %xmm2
736; AVX-NEXT:    vmovq %xmm2, (%rdx)
737; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]
738; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
739; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
740; AVX-NEXT:    vmovq %xmm0, (%rcx)
741; AVX-NEXT:    retq
742  %wide.vec = load <24 x i8>, <24 x i8>* %p, align 4
743  %s1 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
744  %s2 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
745  %s3 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
746  store <8 x i8> %s1, <8 x i8>* %q1, align 4
747  store <8 x i8> %s2, <8 x i8>* %q2, align 4
748  store <8 x i8> %s3, <8 x i8>* %q3, align 4
749  ret void
750}
751
752define void @interleave_24i8_in(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8 x i8>* %q3) nounwind {
753; SSE2-LABEL: interleave_24i8_in:
754; SSE2:       # BB#0:
755; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
756; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
757; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
758; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
759; SSE2-NEXT:    pxor %xmm2, %xmm2
760; SSE2-NEXT:    movdqa %xmm1, %xmm3
761; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
762; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,2,2]
763; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [65535,65535,0,65535,65535,0,65535,65535]
764; SSE2-NEXT:    pand %xmm5, %xmm4
765; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
766; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[0,1,3,3,4,5,6,7]
767; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,6,7]
768; SSE2-NEXT:    pandn %xmm2, %xmm5
769; SSE2-NEXT:    por %xmm4, %xmm5
770; SSE2-NEXT:    movdqa %xmm3, %xmm2
771; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
772; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]
773; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,2,4,5,6,7]
774; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,4,5]
775; SSE2-NEXT:    packuswb %xmm5, %xmm2
776; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,0,255]
777; SSE2-NEXT:    pand %xmm4, %xmm2
778; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,1,0,1]
779; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[0,0,0,3,4,5,6,7]
780; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,6,6]
781; SSE2-NEXT:    pandn %xmm5, %xmm4
782; SSE2-NEXT:    por %xmm2, %xmm4
783; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
784; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
785; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[2,1,1,0,4,5,6,7]
786; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,7]
787; SSE2-NEXT:    packuswb %xmm0, %xmm1
788; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,255,0,255,255,0,255,255,255,255,255,255,255,255]
789; SSE2-NEXT:    pand %xmm2, %xmm1
790; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,1,3,3,4,5,6,7]
791; SSE2-NEXT:    pandn %xmm0, %xmm2
792; SSE2-NEXT:    por %xmm1, %xmm2
793; SSE2-NEXT:    movq %xmm2, 16(%rdi)
794; SSE2-NEXT:    movdqu %xmm4, (%rdi)
795; SSE2-NEXT:    retq
796;
797; SSE42-LABEL: interleave_24i8_in:
798; SSE42:       # BB#0:
799; SSE42-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
800; SSE42-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
801; SSE42-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
802; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
803; SSE42-NEXT:    movdqa %xmm0, %xmm1
804; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,8],zero,xmm1[1,9],zero,xmm1[2,10],zero,xmm1[3,11],zero,xmm1[4,12],zero,xmm1[5]
805; SSE42-NEXT:    movdqa %xmm2, %xmm3
806; SSE42-NEXT:    pshufb {{.*#+}} xmm3 = zero,zero,xmm3[0],zero,zero,xmm3[1],zero,zero,xmm3[2],zero,zero,xmm3[3],zero,zero,xmm3[4],zero
807; SSE42-NEXT:    por %xmm1, %xmm3
808; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[13],zero,xmm0[6,14],zero,xmm0[7,15],zero,xmm0[u,u,u,u,u,u,u,u]
809; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = zero,xmm2[5],zero,zero,xmm2[6],zero,zero,xmm2[7,u,u,u,u,u,u,u,u]
810; SSE42-NEXT:    por %xmm0, %xmm2
811; SSE42-NEXT:    movq %xmm2, 16(%rdi)
812; SSE42-NEXT:    movdqu %xmm3, (%rdi)
813; SSE42-NEXT:    retq
814;
815; AVX-LABEL: interleave_24i8_in:
816; AVX:       # BB#0:
817; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
818; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
819; AVX-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
820; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
821; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[0,8],zero,xmm0[1,9],zero,xmm0[2,10],zero,xmm0[3,11],zero,xmm0[4,12],zero,xmm0[5]
822; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm2[0],zero,zero,xmm2[1],zero,zero,xmm2[2],zero,zero,xmm2[3],zero,zero,xmm2[4],zero
823; AVX-NEXT:    vpor %xmm3, %xmm1, %xmm1
824; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[13],zero,xmm0[6,14],zero,xmm0[7,15],zero,xmm0[u,u,u,u,u,u,u,u]
825; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[5],zero,zero,xmm2[6],zero,zero,xmm2[7,u,u,u,u,u,u,u,u]
826; AVX-NEXT:    vpor %xmm2, %xmm0, %xmm0
827; AVX-NEXT:    vmovq %xmm0, 16(%rdi)
828; AVX-NEXT:    vmovdqu %xmm1, (%rdi)
829; AVX-NEXT:    retq
830  %s1 = load <8 x i8>, <8 x i8>* %q1, align 4
831  %s2 = load <8 x i8>, <8 x i8>* %q2, align 4
832  %s3 = load <8 x i8>, <8 x i8>* %q3, align 4
833  %t1 = shufflevector <8 x i8> %s1, <8 x i8> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
834  %t2 = shufflevector <8 x i8> %s3, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
835  %interleaved = shufflevector <16 x i8> %t1, <16 x i8> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
836  store <24 x i8> %interleaved, <24 x i8>* %p, align 4
837  ret void
838}
839
840
841define void @interleave_24i16_out(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2, <8 x i16>* %q3) nounwind {
842; SSE2-LABEL: interleave_24i16_out:
843; SSE2:       # BB#0:
844; SSE2-NEXT:    movdqu (%rdi), %xmm3
845; SSE2-NEXT:    movdqu 16(%rdi), %xmm2
846; SSE2-NEXT:    movdqu 32(%rdi), %xmm8
847; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,65535,65535,0,65535,65535,0]
848; SSE2-NEXT:    movdqa %xmm3, %xmm4
849; SSE2-NEXT:    pand %xmm1, %xmm4
850; SSE2-NEXT:    pandn %xmm2, %xmm1
851; SSE2-NEXT:    por %xmm4, %xmm1
852; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
853; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]
854; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
855; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7]
856; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,7]
857; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[0,1,2,1]
858; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
859; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,0],xmm1[2,0]
860; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,0]
861; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [65535,65535,0,65535,65535,0,65535,65535]
862; SSE2-NEXT:    movdqa %xmm4, %xmm5
863; SSE2-NEXT:    pandn %xmm2, %xmm5
864; SSE2-NEXT:    movdqa %xmm3, %xmm6
865; SSE2-NEXT:    pand %xmm4, %xmm6
866; SSE2-NEXT:    por %xmm5, %xmm6
867; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm6[2,1,2,3,4,5,6,7]
868; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,4,7]
869; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3]
870; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[1,2,3,0,4,5,6,7]
871; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,6,7]
872; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [65535,65535,65535,65535,65535,0,0,0]
873; SSE2-NEXT:    pand %xmm6, %xmm5
874; SSE2-NEXT:    pshuflw {{.*#+}} xmm7 = xmm8[0,3,2,3,4,5,6,7]
875; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[0,1,0,3]
876; SSE2-NEXT:    pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,4,5,6]
877; SSE2-NEXT:    movdqa %xmm6, %xmm0
878; SSE2-NEXT:    pandn %xmm7, %xmm0
879; SSE2-NEXT:    por %xmm5, %xmm0
880; SSE2-NEXT:    pand %xmm4, %xmm2
881; SSE2-NEXT:    pandn %xmm3, %xmm4
882; SSE2-NEXT:    por %xmm2, %xmm4
883; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[3,1,2,0]
884; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]
885; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,1,0,3]
886; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[2,1,0,3,4,5,6,7]
887; SSE2-NEXT:    pand %xmm6, %xmm2
888; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm8[0,1,2,3,4,7,6,7]
889; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0]
890; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,4,5]
891; SSE2-NEXT:    pandn %xmm3, %xmm6
892; SSE2-NEXT:    por %xmm2, %xmm6
893; SSE2-NEXT:    movups %xmm1, (%rsi)
894; SSE2-NEXT:    movdqu %xmm0, (%rdx)
895; SSE2-NEXT:    movdqu %xmm6, (%rcx)
896; SSE2-NEXT:    retq
897;
898; SSE42-LABEL: interleave_24i16_out:
899; SSE42:       # BB#0:
900; SSE42-NEXT:    movdqu (%rdi), %xmm0
901; SSE42-NEXT:    movdqu 16(%rdi), %xmm1
902; SSE42-NEXT:    movdqu 32(%rdi), %xmm2
903; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,1,2,1]
904; SSE42-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6,5]
905; SSE42-NEXT:    movdqa %xmm0, %xmm4
906; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0],xmm1[1],xmm4[2,3],xmm1[4],xmm4[5,6],xmm1[7]
907; SSE42-NEXT:    pshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
908; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5],xmm3[6,7]
909; SSE42-NEXT:    movdqa %xmm2, %xmm3
910; SSE42-NEXT:    pshufb {{.*#+}} xmm3 = xmm3[0,1,6,7,4,5,6,7,0,1,0,1,6,7,12,13]
911; SSE42-NEXT:    movdqa %xmm0, %xmm5
912; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm5[0,1],xmm1[2],xmm5[3,4],xmm1[5],xmm5[6,7]
913; SSE42-NEXT:    pshufb {{.*#+}} xmm5 = xmm5[2,3,8,9,14,15,4,5,10,11,10,11,8,9,14,15]
914; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm3[5,6,7]
915; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6,7,8,9,2,3,8,9,14,15]
916; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
917; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
918; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm2[5,6,7]
919; SSE42-NEXT:    movdqu %xmm4, (%rsi)
920; SSE42-NEXT:    movdqu %xmm5, (%rdx)
921; SSE42-NEXT:    movdqu %xmm1, (%rcx)
922; SSE42-NEXT:    retq
923;
924; AVX1-LABEL: interleave_24i16_out:
925; AVX1:       # BB#0:
926; AVX1-NEXT:    vmovdqu 32(%rdi), %xmm0
927; AVX1-NEXT:    vmovups (%rdi), %ymm1
928; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
929; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5,6],xmm2[7]
930; AVX1-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
931; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[0,1,2,1]
932; AVX1-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
933; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm4[6,7]
934; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
935; AVX1-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11,10,11,8,9,14,15]
936; AVX1-NEXT:    vpshufb {{.*#+}} xmm5 = xmm0[0,1,6,7,4,5,6,7,0,1,0,1,6,7,12,13]
937; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm5[5,6,7]
938; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
939; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
940; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,7,8,9,2,3,8,9,14,15]
941; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4],xmm0[5,6,7]
942; AVX1-NEXT:    vmovdqu %xmm3, (%rsi)
943; AVX1-NEXT:    vmovdqu %xmm4, (%rdx)
944; AVX1-NEXT:    vmovdqu %xmm0, (%rcx)
945; AVX1-NEXT:    vzeroupper
946; AVX1-NEXT:    retq
947;
948; AVX2-LABEL: interleave_24i16_out:
949; AVX2:       # BB#0:
950; AVX2-NEXT:    vmovdqu (%rdi), %ymm0
951; AVX2-NEXT:    vmovdqu 32(%rdi), %xmm1
952; AVX2-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
953; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
954; AVX2-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6],xmm3[7]
955; AVX2-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11]
956; AVX2-NEXT:    vpblendw {{.*#+}} ymm3 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]
957; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
958; AVX2-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]
959; AVX2-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]
960; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7],ymm0[8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14],ymm1[15]
961; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
962; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
963; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]
964; AVX2-NEXT:    vmovdqu %xmm2, (%rsi)
965; AVX2-NEXT:    vmovdqu %xmm3, (%rdx)
966; AVX2-NEXT:    vmovdqu %xmm0, (%rcx)
967; AVX2-NEXT:    vzeroupper
968; AVX2-NEXT:    retq
969  %wide.vec = load <24 x i16>, <24 x i16>* %p, align 4
970  %s1 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
971  %s2 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
972  %s3 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
973  store <8 x i16> %s1, <8 x i16>* %q1, align 4
974  store <8 x i16> %s2, <8 x i16>* %q2, align 4
975  store <8 x i16> %s3, <8 x i16>* %q3, align 4
976  ret void
977}
978
979define void @interleave_24i16_in(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2, <8 x i16>* %q3) nounwind {
980; SSE2-LABEL: interleave_24i16_in:
981; SSE2:       # BB#0:
982; SSE2-NEXT:    movdqu (%rsi), %xmm3
983; SSE2-NEXT:    movdqu (%rdx), %xmm2
984; SSE2-NEXT:    movdqu (%rcx), %xmm1
985; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,0,3]
986; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [65535,65535,0,65535,65535,0,65535,65535]
987; SSE2-NEXT:    movdqa %xmm0, %xmm5
988; SSE2-NEXT:    pandn %xmm4, %xmm5
989; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,3,3,3]
990; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,1,2,2]
991; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
992; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]
993; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,1,2,2,4,5,6,7]
994; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,4,5]
995; SSE2-NEXT:    pand %xmm0, %xmm3
996; SSE2-NEXT:    por %xmm5, %xmm3
997; SSE2-NEXT:    movdqa %xmm0, %xmm5
998; SSE2-NEXT:    pandn %xmm4, %xmm5
999; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,1,3,3,4,5,6,7]
1000; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
1001; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,1,2,3]
1002; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[2,3,2,0,4,5,6,7]
1003; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,7]
1004; SSE2-NEXT:    pand %xmm0, %xmm2
1005; SSE2-NEXT:    por %xmm5, %xmm2
1006; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [65535,0,65535,65535,0,65535,65535,0]
1007; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,2]
1008; SSE2-NEXT:    pand %xmm5, %xmm1
1009; SSE2-NEXT:    pandn %xmm6, %xmm5
1010; SSE2-NEXT:    por %xmm1, %xmm5
1011; SSE2-NEXT:    pand %xmm0, %xmm5
1012; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm4[0,1,2,3,4,4,6,7]
1013; SSE2-NEXT:    pandn %xmm1, %xmm0
1014; SSE2-NEXT:    por %xmm5, %xmm0
1015; SSE2-NEXT:    movdqu %xmm0, 16(%rdi)
1016; SSE2-NEXT:    movdqu %xmm2, 32(%rdi)
1017; SSE2-NEXT:    movdqu %xmm3, (%rdi)
1018; SSE2-NEXT:    retq
1019;
1020; SSE42-LABEL: interleave_24i16_in:
1021; SSE42:       # BB#0:
1022; SSE42-NEXT:    movdqu (%rsi), %xmm0
1023; SSE42-NEXT:    movdqu (%rdx), %xmm1
1024; SSE42-NEXT:    movdqu (%rcx), %xmm2
1025; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,2]
1026; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,3,3,3]
1027; SSE42-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1028; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
1029; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,0,3]
1030; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm0[0,1],xmm5[2],xmm0[3,4],xmm5[5],xmm0[6,7]
1031; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,2]
1032; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6],xmm3[7]
1033; SSE42-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[0,1,3,3,4,5,6,7]
1034; SSE42-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,6,7]
1035; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm0[0,1],xmm3[2],xmm0[3,4],xmm3[5],xmm0[6,7]
1036; SSE42-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1037; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[4,5,6,7,4,5,8,9,10,11,10,11,12,13,14,15]
1038; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm1[0,1],xmm4[2],xmm1[3,4],xmm4[5],xmm1[6,7]
1039; SSE42-NEXT:    movdqu %xmm4, 32(%rdi)
1040; SSE42-NEXT:    movdqu %xmm3, 16(%rdi)
1041; SSE42-NEXT:    movdqu %xmm5, (%rdi)
1042; SSE42-NEXT:    retq
1043;
1044; AVX1-LABEL: interleave_24i16_in:
1045; AVX1:       # BB#0:
1046; AVX1-NEXT:    vmovdqu (%rsi), %xmm0
1047; AVX1-NEXT:    vmovdqu (%rdx), %xmm1
1048; AVX1-NEXT:    vmovdqu (%rcx), %xmm2
1049; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,2]
1050; AVX1-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm1[0,1,3,3,4,5,6,7]
1051; AVX1-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,4,6,7]
1052; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]
1053; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[1,1,2,2]
1054; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1,2],xmm4[3],xmm3[4,5],xmm4[6],xmm3[7]
1055; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1056; AVX1-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
1057; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[0,0,0,3]
1058; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
1059; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
1060; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1061; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]
1062; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]
1063; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
1064; AVX1-NEXT:    vmovdqu %xmm0, 32(%rdi)
1065; AVX1-NEXT:    vmovups %ymm3, (%rdi)
1066; AVX1-NEXT:    vzeroupper
1067; AVX1-NEXT:    retq
1068;
1069; AVX2-LABEL: interleave_24i16_in:
1070; AVX2:       # BB#0:
1071; AVX2-NEXT:    vmovdqu (%rsi), %xmm0
1072; AVX2-NEXT:    vmovdqu (%rdx), %xmm1
1073; AVX2-NEXT:    vmovdqu (%rcx), %xmm2
1074; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm3
1075; AVX2-NEXT:    vpshufb {{.*#+}} ymm4 = ymm3[0,1,2,3,6,7,2,3,8,9,8,9,4,5,6,7,16,17,18,19,22,23,18,19,24,25,24,25,20,21,22,23]
1076; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm3[2,3,0,1]
1077; AVX2-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,0,1,u,u,u,u,2,3,u,u,u,u,4,5,u,u,22,23,u,u,u,u,24,25,u,u,u,u,26,27]
1078; AVX2-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7],ymm4[8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13,14],ymm3[15]
1079; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = <u,0,0,u,1,1,u,2>
1080; AVX2-NEXT:    vpermd %ymm2, %ymm4, %ymm4
1081; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255]
1082; AVX2-NEXT:    vpblendvb %ymm5, %ymm3, %ymm4, %ymm3
1083; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1084; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]
1085; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]
1086; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
1087; AVX2-NEXT:    vmovdqu %xmm0, 32(%rdi)
1088; AVX2-NEXT:    vmovdqu %ymm3, (%rdi)
1089; AVX2-NEXT:    vzeroupper
1090; AVX2-NEXT:    retq
1091  %s1 = load <8 x i16>, <8 x i16>* %q1, align 4
1092  %s2 = load <8 x i16>, <8 x i16>* %q2, align 4
1093  %s3 = load <8 x i16>, <8 x i16>* %q3, align 4
1094  %t1 = shufflevector <8 x i16> %s1, <8 x i16> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
1095  %t2 = shufflevector <8 x i16> %s3, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1096  %interleaved = shufflevector <16 x i16> %t1, <16 x i16> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
1097  store <24 x i16> %interleaved, <24 x i16>* %p, align 4
1098  ret void
1099}
1100
1101define void @interleave_24i32_out(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2, <8 x i32>* %q3) nounwind {
1102; SSE2-LABEL: interleave_24i32_out:
1103; SSE2:       # BB#0:
1104; SSE2-NEXT:    movdqu 80(%rdi), %xmm8
1105; SSE2-NEXT:    movdqu 64(%rdi), %xmm10
1106; SSE2-NEXT:    movdqu (%rdi), %xmm0
1107; SSE2-NEXT:    movdqu 16(%rdi), %xmm7
1108; SSE2-NEXT:    movdqu 32(%rdi), %xmm9
1109; SSE2-NEXT:    movdqu 48(%rdi), %xmm2
1110; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,1,0,3]
1111; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm7[1]
1112; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm9[0,1,0,1]
1113; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[3,0],xmm3[2,0]
1114; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm6[2,0]
1115; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,1,0,3]
1116; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm6 = xmm6[1],xmm10[1]
1117; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm8[0,1,0,1]
1118; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,0],xmm6[2,0]
1119; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,1],xmm1[2,0]
1120; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1121; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm7[0,0]
1122; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm7[3,3]
1123; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm9[0,1,2,2]
1124; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,0],xmm0[2,0]
1125; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,0]
1126; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
1127; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,0],xmm10[0,0]
1128; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm10[3,3]
1129; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[0,1,2,2]
1130; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,0],xmm2[2,0]
1131; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,0]
1132; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,2,3]
1133; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
1134; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[0,1,0,3]
1135; SSE2-NEXT:    movsd {{.*#+}} xmm4 = xmm1[0],xmm4[1]
1136; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm10[1,1,2,3]
1137; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
1138; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm8[0,1,0,3]
1139; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm5[0],xmm1[1]
1140; SSE2-NEXT:    movups %xmm6, 16(%rsi)
1141; SSE2-NEXT:    movups %xmm3, (%rsi)
1142; SSE2-NEXT:    movups %xmm2, 16(%rdx)
1143; SSE2-NEXT:    movups %xmm0, (%rdx)
1144; SSE2-NEXT:    movupd %xmm1, 16(%rcx)
1145; SSE2-NEXT:    movupd %xmm4, (%rcx)
1146; SSE2-NEXT:    retq
1147;
1148; SSE42-LABEL: interleave_24i32_out:
1149; SSE42:       # BB#0:
1150; SSE42-NEXT:    movdqu 80(%rdi), %xmm8
1151; SSE42-NEXT:    movdqu 64(%rdi), %xmm1
1152; SSE42-NEXT:    movdqu (%rdi), %xmm5
1153; SSE42-NEXT:    movdqu 16(%rdi), %xmm6
1154; SSE42-NEXT:    movdqu 32(%rdi), %xmm2
1155; SSE42-NEXT:    movdqu 48(%rdi), %xmm4
1156; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,3,2,3]
1157; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1,2,3],xmm6[4,5,6,7]
1158; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]
1159; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm7[0,1,2,3,4,5],xmm3[6,7]
1160; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,3,2,3]
1161; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1,2,3],xmm1[4,5,6,7]
1162; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[0,1,0,1]
1163; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm7[0,1,2,3,4,5],xmm0[6,7]
1164; SSE42-NEXT:    pshufd {{.*#+}} xmm9 = xmm2[0,1,2,2]
1165; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[2,3,0,1]
1166; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm6[2,3],xmm7[4,5,6,7]
1167; SSE42-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
1168; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,0,3,3]
1169; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5],xmm9[6,7]
1170; SSE42-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[2,3,0,1]
1171; SSE42-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1],xmm1[2,3],xmm6[4,5,6,7]
1172; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3],xmm1[4,5,6,7]
1173; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,0,3,3]
1174; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[0,1,2,2]
1175; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm1[0,1,2,3,4,5],xmm4[6,7]
1176; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,1,0,3]
1177; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm7[0,1,2,3],xmm1[4,5,6,7]
1178; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm8[0,1,0,3]
1179; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm6[0,1,2,3],xmm2[4,5,6,7]
1180; SSE42-NEXT:    movdqu %xmm0, 16(%rsi)
1181; SSE42-NEXT:    movdqu %xmm3, (%rsi)
1182; SSE42-NEXT:    movdqu %xmm4, 16(%rdx)
1183; SSE42-NEXT:    movdqu %xmm5, (%rdx)
1184; SSE42-NEXT:    movdqu %xmm2, 16(%rcx)
1185; SSE42-NEXT:    movdqu %xmm1, (%rcx)
1186; SSE42-NEXT:    retq
1187;
1188; AVX1-LABEL: interleave_24i32_out:
1189; AVX1:       # BB#0:
1190; AVX1-NEXT:    vmovups (%rdi), %ymm0
1191; AVX1-NEXT:    vmovups 32(%rdi), %ymm1
1192; AVX1-NEXT:    vmovups 64(%rdi), %ymm2
1193; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
1194; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = zero,zero,xmm2[2],xmm3[1]
1195; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm4
1196; AVX1-NEXT:    vblendps {{.*#+}} ymm5 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
1197; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm6
1198; AVX1-NEXT:    vblendps {{.*#+}} xmm5 = xmm5[0,1],xmm6[2],xmm5[3]
1199; AVX1-NEXT:    vpermilps {{.*#+}} xmm5 = xmm5[0,3,2,1]
1200; AVX1-NEXT:    vpermilps {{.*#+}} xmm6 = xmm6[0,3,2,3]
1201; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm5
1202; AVX1-NEXT:    vblendpd {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3]
1203; AVX1-NEXT:    vblendps {{.*#+}} xmm5 = xmm2[0,1],xmm3[2],xmm2[3]
1204; AVX1-NEXT:    vpermilps {{.*#+}} xmm5 = xmm5[0,0,3,2]
1205; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm5
1206; AVX1-NEXT:    vblendps {{.*#+}} ymm6 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1207; AVX1-NEXT:    vextractf128 $1, %ymm6, %xmm7
1208; AVX1-NEXT:    vblendps {{.*#+}} xmm6 = xmm7[0],xmm6[1,2],xmm7[3]
1209; AVX1-NEXT:    vpermilps {{.*#+}} xmm6 = xmm6[1,0,3,2]
1210; AVX1-NEXT:    vmovshdup {{.*#+}} xmm7 = xmm7[1,1,3,3]
1211; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm6, %ymm6
1212; AVX1-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4],ymm5[5,6,7]
1213; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0,3]
1214; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm2
1215; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
1216; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1217; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
1218; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,1,0,3]
1219; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
1220; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1221; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
1222; AVX1-NEXT:    vmovupd %ymm4, (%rsi)
1223; AVX1-NEXT:    vmovups %ymm5, (%rdx)
1224; AVX1-NEXT:    vmovups %ymm0, (%rcx)
1225; AVX1-NEXT:    vzeroupper
1226; AVX1-NEXT:    retq
1227;
1228; AVX2-LABEL: interleave_24i32_out:
1229; AVX2:       # BB#0:
1230; AVX2-NEXT:    vmovdqu (%rdi), %ymm0
1231; AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
1232; AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
1233; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = <u,u,u,u,u,u,2,5>
1234; AVX2-NEXT:    vpermd %ymm2, %ymm3, %ymm3
1235; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
1236; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = <0,3,6,1,4,7,u,u>
1237; AVX2-NEXT:    vpermd %ymm4, %ymm5, %ymm4
1238; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
1239; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = <u,u,u,u,u,0,3,6>
1240; AVX2-NEXT:    vpermd %ymm2, %ymm4, %ymm4
1241; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1242; AVX2-NEXT:    vmovdqa {{.*#+}} ymm6 = <1,4,7,2,5,u,u,u>
1243; AVX2-NEXT:    vpermd %ymm5, %ymm6, %ymm5
1244; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5,6,7]
1245; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
1246; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = <2,5,0,3,6,u,u,u>
1247; AVX2-NEXT:    vpermd %ymm0, %ymm1, %ymm0
1248; AVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm2[0,1,0,3,4,5,4,7]
1249; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,3]
1250; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
1251; AVX2-NEXT:    vmovdqu %ymm3, (%rsi)
1252; AVX2-NEXT:    vmovdqu %ymm4, (%rdx)
1253; AVX2-NEXT:    vmovdqu %ymm0, (%rcx)
1254; AVX2-NEXT:    vzeroupper
1255; AVX2-NEXT:    retq
1256  %wide.vec = load <24 x i32>, <24 x i32>* %p, align 4
1257  %s1 = shufflevector <24 x i32> %wide.vec, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
1258  %s2 = shufflevector <24 x i32> %wide.vec, <24 x i32> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
1259  %s3 = shufflevector <24 x i32> %wide.vec, <24 x i32> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
1260  store <8 x i32> %s1, <8 x i32>* %q1, align 4
1261  store <8 x i32> %s2, <8 x i32>* %q2, align 4
1262  store <8 x i32> %s3, <8 x i32>* %q3, align 4
1263  ret void
1264}
1265
1266define void @interleave_24i32_in(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2, <8 x i32>* %q3) nounwind {
1267; SSE2-LABEL: interleave_24i32_in:
1268; SSE2:       # BB#0:
1269; SSE2-NEXT:    movdqu (%rsi), %xmm5
1270; SSE2-NEXT:    movdqu 16(%rsi), %xmm2
1271; SSE2-NEXT:    movdqu (%rdx), %xmm6
1272; SSE2-NEXT:    movdqu 16(%rdx), %xmm1
1273; SSE2-NEXT:    movdqu (%rcx), %xmm7
1274; SSE2-NEXT:    movdqu 16(%rcx), %xmm4
1275; SSE2-NEXT:    movdqa %xmm5, %xmm0
1276; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
1277; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,2,2]
1278; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,1,0,1]
1279; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm0[3,0]
1280; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[0,2]
1281; SSE2-NEXT:    movdqa %xmm7, %xmm3
1282; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,0],xmm6[1,0]
1283; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm6[2,2]
1284; SSE2-NEXT:    punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm5[2],xmm7[3],xmm5[3]
1285; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm3[3,0]
1286; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm5[0,2]
1287; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,3,2,2]
1288; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]
1289; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[2,0],xmm5[3,0]
1290; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm6[0,2]
1291; SSE2-NEXT:    movdqa %xmm2, %xmm6
1292; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
1293; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[0,1,2,2]
1294; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,1,0,1]
1295; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[2,0],xmm6[3,0]
1296; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,1],xmm7[0,2]
1297; SSE2-NEXT:    movdqa %xmm4, %xmm7
1298; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[1,0],xmm1[1,0]
1299; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[2,0],xmm1[2,2]
1300; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
1301; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm7[3,0]
1302; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,1],xmm2[0,2]
1303; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,3,2,2]
1304; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
1305; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,0]
1306; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2]
1307; SSE2-NEXT:    movups %xmm2, 80(%rdi)
1308; SSE2-NEXT:    movups %xmm7, 64(%rdi)
1309; SSE2-NEXT:    movups %xmm6, 48(%rdi)
1310; SSE2-NEXT:    movups %xmm5, 32(%rdi)
1311; SSE2-NEXT:    movups %xmm3, 16(%rdi)
1312; SSE2-NEXT:    movups %xmm0, (%rdi)
1313; SSE2-NEXT:    retq
1314;
1315; SSE42-LABEL: interleave_24i32_in:
1316; SSE42:       # BB#0:
1317; SSE42-NEXT:    movdqu (%rsi), %xmm5
1318; SSE42-NEXT:    movdqu 16(%rsi), %xmm2
1319; SSE42-NEXT:    movdqu (%rdx), %xmm6
1320; SSE42-NEXT:    movdqu 16(%rdx), %xmm1
1321; SSE42-NEXT:    movdqu (%rcx), %xmm7
1322; SSE42-NEXT:    movdqu 16(%rcx), %xmm4
1323; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,1,1]
1324; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,1,0,1]
1325; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm0[2,3],xmm3[4,5,6,7]
1326; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,1,0,1]
1327; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5],xmm3[6,7]
1328; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,2,2]
1329; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm7[2,3],xmm3[4,5,6,7]
1330; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm5[4,5],xmm3[6,7]
1331; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,0,1]
1332; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[2,3,2,3]
1333; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm5[2,3],xmm7[4,5,6,7]
1334; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[2,2,3,3]
1335; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm7[0,1,2,3],xmm5[4,5],xmm7[6,7]
1336; SSE42-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,1,1]
1337; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[0,1,0,1]
1338; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm6[2,3],xmm7[4,5,6,7]
1339; SSE42-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,1,0,1]
1340; SSE42-NEXT:    pblendw {{.*#+}} xmm6 = xmm7[0,1,2,3],xmm6[4,5],xmm7[6,7]
1341; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,2,2]
1342; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm4[2,3],xmm7[4,5,6,7]
1343; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1,2,3],xmm2[4,5],xmm7[6,7]
1344; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
1345; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
1346; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3],xmm4[4,5,6,7]
1347; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
1348; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm4[0,1,2,3],xmm1[4,5],xmm4[6,7]
1349; SSE42-NEXT:    movdqu %xmm1, 80(%rdi)
1350; SSE42-NEXT:    movdqu %xmm7, 64(%rdi)
1351; SSE42-NEXT:    movdqu %xmm6, 48(%rdi)
1352; SSE42-NEXT:    movdqu %xmm5, 32(%rdi)
1353; SSE42-NEXT:    movdqu %xmm3, 16(%rdi)
1354; SSE42-NEXT:    movdqu %xmm0, (%rdi)
1355; SSE42-NEXT:    retq
1356;
1357; AVX1-LABEL: interleave_24i32_in:
1358; AVX1:       # BB#0:
1359; AVX1-NEXT:    vmovups (%rsi), %ymm0
1360; AVX1-NEXT:    vmovups (%rdx), %ymm1
1361; AVX1-NEXT:    vmovupd (%rcx), %ymm2
1362; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm0[2,0],xmm1[2,0]
1363; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[1,1],xmm3[0,2]
1364; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[0,0],xmm0[0,0]
1365; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[2,0],xmm0[2,1]
1366; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
1367; AVX1-NEXT:    vmovddup {{.*#+}} xmm4 = xmm2[0,0]
1368; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm4, %ymm4
1369; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
1370; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
1371; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
1372; AVX1-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[3,0],xmm4[3,0]
1373; AVX1-NEXT:    vshufps {{.*#+}} xmm6 = xmm4[2,1],xmm6[0,2]
1374; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[1,0],xmm5[1,0]
1375; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[2,0],xmm5[2,2]
1376; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm4
1377; AVX1-NEXT:    vpermilpd {{.*#+}} ymm5 = ymm0[1,1,3,3]
1378; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm5[2,3,2,3]
1379; AVX1-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]
1380; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,2]
1381; AVX1-NEXT:    vpermilpd {{.*#+}} ymm2 = ymm2[1,1,2,2]
1382; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
1383; AVX1-NEXT:    vpermilps {{.*#+}} ymm1 = ymm1[0,0,3,3,4,4,7,7]
1384; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1385; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
1386; AVX1-NEXT:    vmovups %ymm4, 64(%rdi)
1387; AVX1-NEXT:    vmovups %ymm3, (%rdi)
1388; AVX1-NEXT:    vzeroupper
1389; AVX1-NEXT:    retq
1390;
1391; AVX2-LABEL: interleave_24i32_in:
1392; AVX2:       # BB#0:
1393; AVX2-NEXT:    vmovdqu (%rsi), %ymm0
1394; AVX2-NEXT:    vmovdqu (%rdx), %ymm1
1395; AVX2-NEXT:    vmovdqu (%rcx), %ymm2
1396; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,0,2,2]
1397; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm3[0,1,0,1]
1398; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[0,0,2,1]
1399; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7]
1400; AVX2-NEXT:    vpbroadcastq %xmm2, %ymm4
1401; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
1402; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,1,3,3]
1403; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[1,2,3,3,5,6,7,7]
1404; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
1405; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6],ymm4[7]
1406; AVX2-NEXT:    vpbroadcastq 24(%rsi), %ymm5
1407; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]
1408; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,1,2,2]
1409; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,1,2,2]
1410; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
1411; AVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,3,3,4,4,7,7]
1412; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1413; AVX2-NEXT:    vmovdqu %ymm0, 32(%rdi)
1414; AVX2-NEXT:    vmovdqu %ymm4, 64(%rdi)
1415; AVX2-NEXT:    vmovdqu %ymm3, (%rdi)
1416; AVX2-NEXT:    vzeroupper
1417; AVX2-NEXT:    retq
1418  %s1 = load <8 x i32>, <8 x i32>* %q1, align 4
1419  %s2 = load <8 x i32>, <8 x i32>* %q2, align 4
1420  %s3 = load <8 x i32>, <8 x i32>* %q3, align 4
1421  %t1 = shufflevector <8 x i32> %s1, <8 x i32> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
1422  %t2 = shufflevector <8 x i32> %s3, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1423  %interleaved = shufflevector <16 x i32> %t1, <16 x i32> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
1424  store <24 x i32> %interleaved, <24 x i32>* %p, align 4
1425  ret void
1426}
1427
1428define <2 x double> @wrongorder(<4 x double> %A, <8 x double>* %P) #0 {
1429; SSE2-LABEL: wrongorder:
1430; SSE2:       # BB#0:
1431; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0,0]
1432; SSE2-NEXT:    movaps %xmm0, 48(%rdi)
1433; SSE2-NEXT:    movaps %xmm0, 32(%rdi)
1434; SSE2-NEXT:    movaps %xmm0, 16(%rdi)
1435; SSE2-NEXT:    movaps %xmm0, (%rdi)
1436; SSE2-NEXT:    retq
1437;
1438; SSE42-LABEL: wrongorder:
1439; SSE42:       # BB#0:
1440; SSE42-NEXT:    movddup {{.*#+}} xmm0 = xmm0[0,0]
1441; SSE42-NEXT:    movapd %xmm0, 48(%rdi)
1442; SSE42-NEXT:    movapd %xmm0, 32(%rdi)
1443; SSE42-NEXT:    movapd %xmm0, 16(%rdi)
1444; SSE42-NEXT:    movapd %xmm0, (%rdi)
1445; SSE42-NEXT:    retq
1446;
1447; AVX1-LABEL: wrongorder:
1448; AVX1:       # BB#0:
1449; AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
1450; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
1451; AVX1-NEXT:    vmovapd %ymm1, 32(%rdi)
1452; AVX1-NEXT:    vmovapd %ymm1, (%rdi)
1453; AVX1-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
1454; AVX1-NEXT:    vzeroupper
1455; AVX1-NEXT:    retq
1456;
1457; AVX2-LABEL: wrongorder:
1458; AVX2:       # BB#0:
1459; AVX2-NEXT:    vbroadcastsd %xmm0, %ymm1
1460; AVX2-NEXT:    vmovapd %ymm1, 32(%rdi)
1461; AVX2-NEXT:    vmovapd %ymm1, (%rdi)
1462; AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
1463; AVX2-NEXT:    vzeroupper
1464; AVX2-NEXT:    retq
1465  %shuffle = shufflevector <4 x double> %A, <4 x double> %A, <8 x i32> zeroinitializer
1466  store <8 x double> %shuffle, <8 x double>* %P, align 64
1467  %m2 = load <8 x double>, <8 x double>* %P, align 64
1468  store <8 x double> %m2, <8 x double>* %P, align 64
1469  %m3 = load <8 x double>, <8 x double>* %P, align 64
1470  %m4 = shufflevector <8 x double> %m3, <8 x double> undef, <2 x i32> <i32 2, i32 0>
1471  ret <2 x double> %m4
1472}
1473