1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
3; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+sse4.2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE42
4; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
5; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
6
7define void @v3i64(<2 x i64> %a, <2 x i64> %b, <3 x i64>* %p) nounwind {
8; SSE2-LABEL: v3i64:
9; SSE2:       # BB#0:
10; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
11; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
12; SSE2-NEXT:    movq %xmm2, 16(%rdi)
13; SSE2-NEXT:    movdqa %xmm0, (%rdi)
14; SSE2-NEXT:    retq
15;
16; SSE42-LABEL: v3i64:
17; SSE42:       # BB#0:
18; SSE42-NEXT:    pextrq $1, %xmm0, 16(%rdi)
19; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
20; SSE42-NEXT:    movdqa %xmm0, (%rdi)
21; SSE42-NEXT:    retq
22;
23; AVX1-LABEL: v3i64:
24; AVX1:       # BB#0:
25; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0]
26; AVX1-NEXT:    vpextrq $1, %xmm0, 16(%rdi)
27; AVX1-NEXT:    vmovapd %xmm1, (%rdi)
28; AVX1-NEXT:    retq
29;
30; AVX2-LABEL: v3i64:
31; AVX2:       # BB#0:
32; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<def>
33; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
34; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
35; AVX2-NEXT:    vpextrq $1, %xmm0, 16(%rdi)
36; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
37; AVX2-NEXT:    vzeroupper
38; AVX2-NEXT:    retq
39  %r = shufflevector <2 x i64> %a, <2 x i64> %b, <3 x i32> <i32 0, i32 2, i32 1>
40  store <3 x i64> %r, <3 x i64>* %p
41  ret void
42}
43define void @v3f64(<2 x double> %a, <2 x double> %b, <3 x double>* %p) nounwind {
44; SSE-LABEL: v3f64:
45; SSE:       # BB#0:
46; SSE-NEXT:    movhpd %xmm0, 16(%rdi)
47; SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
48; SSE-NEXT:    movapd %xmm0, (%rdi)
49; SSE-NEXT:    retq
50;
51; AVX1-LABEL: v3f64:
52; AVX1:       # BB#0:
53; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0]
54; AVX1-NEXT:    vmovhpd %xmm0, 16(%rdi)
55; AVX1-NEXT:    vmovapd %xmm1, (%rdi)
56; AVX1-NEXT:    retq
57;
58; AVX2-LABEL: v3f64:
59; AVX2:       # BB#0:
60; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<def>
61; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
62; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
63; AVX2-NEXT:    vmovhpd %xmm0, 16(%rdi)
64; AVX2-NEXT:    vmovapd %xmm1, (%rdi)
65; AVX2-NEXT:    vzeroupper
66; AVX2-NEXT:    retq
67  %r = shufflevector <2 x double> %a, <2 x double> %b, <3 x i32> <i32 0, i32 2, i32 1>
68  store <3 x double> %r, <3 x double>* %p
69  ret void
70}
71
72define void @v3i32(<2 x i32> %a, <2 x i32> %b, <3 x i32>* %p) nounwind {
73; SSE2-LABEL: v3i32:
74; SSE2:       # BB#0:
75; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
76; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
77; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
78; SSE2-NEXT:    movd %xmm0, 8(%rdi)
79; SSE2-NEXT:    movq %xmm2, (%rdi)
80; SSE2-NEXT:    retq
81;
82; SSE42-LABEL: v3i32:
83; SSE42:       # BB#0:
84; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
85; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
86; SSE42-NEXT:    pextrd $2, %xmm0, 8(%rdi)
87; SSE42-NEXT:    movq %xmm1, (%rdi)
88; SSE42-NEXT:    retq
89;
90; AVX1-LABEL: v3i32:
91; AVX1:       # BB#0:
92; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
93; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
94; AVX1-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
95; AVX1-NEXT:    vmovq %xmm1, (%rdi)
96; AVX1-NEXT:    retq
97;
98; AVX2-LABEL: v3i32:
99; AVX2:       # BB#0:
100; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
101; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2,3]
102; AVX2-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
103; AVX2-NEXT:    vmovq %xmm1, (%rdi)
104; AVX2-NEXT:    retq
105  %r = shufflevector <2 x i32> %a, <2 x i32> %b, <3 x i32> <i32 0, i32 2, i32 1>
106  store <3 x i32> %r, <3 x i32>* %p
107  ret void
108}
109
110define void @v5i16(<4 x i16> %a, <4 x i16> %b, <5 x i16>* %p) nounwind {
111; SSE2-LABEL: v5i16:
112; SSE2:       # BB#0:
113; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
114; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
115; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
116; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
117; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
118; SSE2-NEXT:    pextrw $6, %xmm0, %eax
119; SSE2-NEXT:    movw %ax, 8(%rdi)
120; SSE2-NEXT:    movq %xmm2, (%rdi)
121; SSE2-NEXT:    retq
122;
123; SSE42-LABEL: v5i16:
124; SSE42:       # BB#0:
125; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
126; SSE42-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
127; SSE42-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
128; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
129; SSE42-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
130; SSE42-NEXT:    pextrw $6, %xmm0, 8(%rdi)
131; SSE42-NEXT:    movq %xmm2, (%rdi)
132; SSE42-NEXT:    retq
133;
134; AVX-LABEL: v5i16:
135; AVX:       # BB#0:
136; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
137; AVX-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
138; AVX-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
139; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
140; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
141; AVX-NEXT:    vpextrw $6, %xmm0, 8(%rdi)
142; AVX-NEXT:    vmovq %xmm1, (%rdi)
143; AVX-NEXT:    retq
144  %r = shufflevector <4 x i16> %a, <4 x i16> %b, <5 x i32> <i32 0, i32 5, i32 1, i32 6, i32 3>
145  store <5 x i16> %r, <5 x i16>* %p
146  ret void
147}
148
149define void @v5i32(<4 x i32> %a, <4 x i32> %b, <5 x i32>* %p) nounwind {
150; SSE2-LABEL: v5i32:
151; SSE2:       # BB#0:
152; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,2,2,3]
153; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
154; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
155; SSE2-NEXT:    movd %xmm2, 16(%rdi)
156; SSE2-NEXT:    movdqa %xmm0, (%rdi)
157; SSE2-NEXT:    retq
158;
159; SSE42-LABEL: v5i32:
160; SSE42:       # BB#0:
161; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
162; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
163; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
164; SSE42-NEXT:    pextrd $3, %xmm0, 16(%rdi)
165; SSE42-NEXT:    movdqa %xmm2, (%rdi)
166; SSE42-NEXT:    retq
167;
168; AVX1-LABEL: v5i32:
169; AVX1:       # BB#0:
170; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[0,1],xmm1[1,2]
171; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,1,3]
172; AVX1-NEXT:    vpextrd $3, %xmm0, 16(%rdi)
173; AVX1-NEXT:    vmovaps %xmm1, (%rdi)
174; AVX1-NEXT:    retq
175;
176; AVX2-LABEL: v5i32:
177; AVX2:       # BB#0:
178; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<def>
179; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
180; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = <0,5,1,6,3,u,u,u>
181; AVX2-NEXT:    vpermd %ymm1, %ymm2, %ymm1
182; AVX2-NEXT:    vpextrd $3, %xmm0, 16(%rdi)
183; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
184; AVX2-NEXT:    vzeroupper
185; AVX2-NEXT:    retq
186  %r = shufflevector <4 x i32> %a, <4 x i32> %b, <5 x i32> <i32 0, i32 5, i32 1, i32 6, i32 3>
187  store <5 x i32> %r, <5 x i32>* %p
188  ret void
189}
190
191define void @v5f32(<4 x float> %a, <4 x float> %b, <5 x float>* %p) nounwind {
192; SSE2-LABEL: v5f32:
193; SSE2:       # BB#0:
194; SSE2-NEXT:    movaps %xmm0, %xmm2
195; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[1,2]
196; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2,1,3]
197; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
198; SSE2-NEXT:    movss %xmm0, 16(%rdi)
199; SSE2-NEXT:    movaps %xmm2, (%rdi)
200; SSE2-NEXT:    retq
201;
202; SSE42-LABEL: v5f32:
203; SSE42:       # BB#0:
204; SSE42-NEXT:    extractps $3, %xmm0, 16(%rdi)
205; SSE42-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[1,2]
206; SSE42-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
207; SSE42-NEXT:    movaps %xmm0, (%rdi)
208; SSE42-NEXT:    retq
209;
210; AVX1-LABEL: v5f32:
211; AVX1:       # BB#0:
212; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[0,1],xmm1[1,2]
213; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,1,3]
214; AVX1-NEXT:    vextractps $3, %xmm0, 16(%rdi)
215; AVX1-NEXT:    vmovaps %xmm1, (%rdi)
216; AVX1-NEXT:    retq
217;
218; AVX2-LABEL: v5f32:
219; AVX2:       # BB#0:
220; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<def>
221; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
222; AVX2-NEXT:    vmovaps {{.*#+}} ymm2 = <0,5,1,6,3,u,u,u>
223; AVX2-NEXT:    vpermps %ymm1, %ymm2, %ymm1
224; AVX2-NEXT:    vextractps $3, %xmm0, 16(%rdi)
225; AVX2-NEXT:    vmovaps %xmm1, (%rdi)
226; AVX2-NEXT:    vzeroupper
227; AVX2-NEXT:    retq
228  %r = shufflevector <4 x float> %a, <4 x float> %b, <5 x i32> <i32 0, i32 5, i32 1, i32 6, i32 3>
229  store <5 x float> %r, <5 x float>* %p
230  ret void
231}
232
233define void @v7i8(<4 x i8> %a, <4 x i8> %b, <7 x i8>* %p) nounwind {
234; SSE2-LABEL: v7i8:
235; SSE2:       # BB#0:
236; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
237; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,65535,0,65535,0,65535,65535,65535]
238; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,0,3]
239; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,0,4,5,6,7]
240; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,4,7]
241; SSE2-NEXT:    pand %xmm2, %xmm1
242; SSE2-NEXT:    pandn %xmm0, %xmm2
243; SSE2-NEXT:    por %xmm1, %xmm2
244; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
245; SSE2-NEXT:    pand %xmm2, %xmm0
246; SSE2-NEXT:    packuswb %xmm0, %xmm0
247; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,1,3]
248; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
249; SSE2-NEXT:    movdqa %xmm2, -{{[0-9]+}}(%rsp)
250; SSE2-NEXT:    movb -{{[0-9]+}}(%rsp), %al
251; SSE2-NEXT:    movb %al, 6(%rdi)
252; SSE2-NEXT:    movd %xmm1, (%rdi)
253; SSE2-NEXT:    pextrw $4, %xmm0, %eax
254; SSE2-NEXT:    movw %ax, 4(%rdi)
255; SSE2-NEXT:    retq
256;
257; SSE42-LABEL: v7i8:
258; SSE42:       # BB#0:
259; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
260; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[8,9,8,9,4,5,8,9,0,1,12,13,0,1,14,15]
261; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5,6,7]
262; SSE42-NEXT:    pextrb $12, %xmm1, 6(%rdi)
263; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
264; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
265; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
266; SSE42-NEXT:    pextrw $4, %xmm1, 4(%rdi)
267; SSE42-NEXT:    movd %xmm0, (%rdi)
268; SSE42-NEXT:    retq
269;
270; AVX-LABEL: v7i8:
271; AVX:       # BB#0:
272; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
273; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[8,9,8,9,4,5,8,9,0,1,12,13,0,1,14,15]
274; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5,6,7]
275; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
276; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
277; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
278; AVX-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
279; AVX-NEXT:    vpextrw $4, %xmm1, 4(%rdi)
280; AVX-NEXT:    vmovd %xmm2, (%rdi)
281; AVX-NEXT:    retq
282  %r = shufflevector <4 x i8> %a, <4 x i8> %b, <7 x i32> <i32 0, i32 6, i32 3, i32 6, i32 1, i32 7, i32 4>
283  store <7 x i8> %r, <7 x i8>* %p
284  ret void
285}
286
287define void @v7i16(<4 x i16> %a, <4 x i16> %b, <7 x i16>* %p) nounwind {
288; SSE2-LABEL: v7i16:
289; SSE2:       # BB#0:
290; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
291; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,65535,0,65535,0,65535,65535,65535]
292; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,1,0,3]
293; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,0,2,0,4,5,6,7]
294; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,4,7]
295; SSE2-NEXT:    pand %xmm2, %xmm3
296; SSE2-NEXT:    pandn %xmm0, %xmm2
297; SSE2-NEXT:    por %xmm3, %xmm2
298; SSE2-NEXT:    movd %xmm1, %eax
299; SSE2-NEXT:    movw %ax, 12(%rdi)
300; SSE2-NEXT:    movq %xmm2, (%rdi)
301; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
302; SSE2-NEXT:    movd %xmm0, 8(%rdi)
303; SSE2-NEXT:    retq
304;
305; SSE42-LABEL: v7i16:
306; SSE42:       # BB#0:
307; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
308; SSE42-NEXT:    pextrw $0, %xmm1, 12(%rdi)
309; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[8,9,8,9,4,5,8,9,0,1,12,13,0,1,14,15]
310; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5,6,7]
311; SSE42-NEXT:    pextrd $2, %xmm1, 8(%rdi)
312; SSE42-NEXT:    movq %xmm1, (%rdi)
313; SSE42-NEXT:    retq
314;
315; AVX-LABEL: v7i16:
316; AVX:       # BB#0:
317; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,3,1,3]
318; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = xmm1[8,9,8,9,4,5,8,9,0,1,12,13,0,1,14,15]
319; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5,6,7]
320; AVX-NEXT:    vpextrw $0, %xmm1, 12(%rdi)
321; AVX-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
322; AVX-NEXT:    vmovq %xmm0, (%rdi)
323; AVX-NEXT:    retq
324  %r = shufflevector <4 x i16> %a, <4 x i16> %b, <7 x i32> <i32 0, i32 6, i32 3, i32 6, i32 1, i32 7, i32 4>
325  store <7 x i16> %r, <7 x i16>* %p
326  ret void
327}
328
329
330define void @v7i32(<4 x i32> %a, <4 x i32> %b, <7 x i32>* %p) nounwind {
331; SSE2-LABEL: v7i32:
332; SSE2:       # BB#0:
333; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,1,2,2]
334; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,1,0,3]
335; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
336; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[3,0]
337; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3]
338; SSE2-NEXT:    movd %xmm1, 24(%rdi)
339; SSE2-NEXT:    movlps %xmm0, 16(%rdi)
340; SSE2-NEXT:    movdqa %xmm3, (%rdi)
341; SSE2-NEXT:    retq
342;
343; SSE42-LABEL: v7i32:
344; SSE42:       # BB#0:
345; SSE42-NEXT:    movdqa %xmm1, %xmm2
346; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3],xmm2[4,5,6,7]
347; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5],xmm0[6,7]
348; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,3,2]
349; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,0,3]
350; SSE42-NEXT:    movd %xmm1, 24(%rdi)
351; SSE42-NEXT:    movq %xmm2, 16(%rdi)
352; SSE42-NEXT:    movdqa %xmm0, (%rdi)
353; SSE42-NEXT:    retq
354;
355; AVX1-LABEL: v7i32:
356; AVX1:       # BB#0:
357; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm0[0,1],xmm1[2],xmm0[3]
358; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[0,2,3,2]
359; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3]
360; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,3,0,3]
361; AVX1-NEXT:    vmovss %xmm1, 24(%rdi)
362; AVX1-NEXT:    vmovlps %xmm0, 16(%rdi)
363; AVX1-NEXT:    vmovaps %xmm2, (%rdi)
364; AVX1-NEXT:    retq
365;
366; AVX2-LABEL: v7i32:
367; AVX2:       # BB#0:
368; AVX2-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<def>
369; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
370; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = <0,6,3,6,1,7,4,u>
371; AVX2-NEXT:    vpermd %ymm0, %ymm2, %ymm0
372; AVX2-NEXT:    vmovd %xmm1, 24(%rdi)
373; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
374; AVX2-NEXT:    vmovq %xmm1, 16(%rdi)
375; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
376; AVX2-NEXT:    vzeroupper
377; AVX2-NEXT:    retq
378  %r = shufflevector <4 x i32> %a, <4 x i32> %b, <7 x i32> <i32 0, i32 6, i32 3, i32 6, i32 1, i32 7, i32 4>
379  store <7 x i32> %r, <7 x i32>* %p
380  ret void
381}
382
383define void @v12i8(<8 x i8> %a, <8 x i8> %b, <12 x i8>* %p) nounwind {
384; SSE2-LABEL: v12i8:
385; SSE2:       # BB#0:
386; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
387; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
388; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
389; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,255,255]
390; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
391; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
392; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
393; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
394; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
395; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,1,4,5,6,7]
396; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,4]
397; SSE2-NEXT:    packuswb %xmm3, %xmm0
398; SSE2-NEXT:    pand %xmm2, %xmm0
399; SSE2-NEXT:    pandn %xmm1, %xmm2
400; SSE2-NEXT:    por %xmm0, %xmm2
401; SSE2-NEXT:    movq %xmm2, (%rdi)
402; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
403; SSE2-NEXT:    movd %xmm0, 8(%rdi)
404; SSE2-NEXT:    retq
405;
406; SSE42-LABEL: v12i8:
407; SSE42:       # BB#0:
408; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = zero,zero,xmm1[0],zero,zero,xmm1[2],zero,zero,xmm1[4],zero,zero,xmm1[6,u,u,u,u]
409; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8],zero,xmm0[2,10],zero,xmm0[4,12],zero,xmm0[6,14],zero,xmm0[u,u,u,u]
410; SSE42-NEXT:    por %xmm1, %xmm0
411; SSE42-NEXT:    pextrd $2, %xmm0, 8(%rdi)
412; SSE42-NEXT:    movq %xmm0, (%rdi)
413; SSE42-NEXT:    retq
414;
415; AVX-LABEL: v12i8:
416; AVX:       # BB#0:
417; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,xmm1[0],zero,zero,xmm1[2],zero,zero,xmm1[4],zero,zero,xmm1[6,u,u,u,u]
418; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8],zero,xmm0[2,10],zero,xmm0[4,12],zero,xmm0[6,14],zero,xmm0[u,u,u,u]
419; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
420; AVX-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
421; AVX-NEXT:    vmovq %xmm0, (%rdi)
422; AVX-NEXT:    retq
423  %r = shufflevector <8 x i8> %a, <8 x i8> %b, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
424  store <12 x i8> %r, <12 x i8>* %p
425  ret void
426}
427
428define void @v12i16(<8 x i16> %a, <8 x i16> %b, <12 x i16>* %p) nounwind {
429; SSE2-LABEL: v12i16:
430; SSE2:       # BB#0:
431; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,0,3]
432; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [65535,65535,0,65535,65535,0,65535,65535]
433; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,6,5,4,7]
434; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,3,2,1]
435; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,2,2,1,4,5,6,7]
436; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,6,4]
437; SSE2-NEXT:    pand %xmm3, %xmm4
438; SSE2-NEXT:    pandn %xmm2, %xmm3
439; SSE2-NEXT:    por %xmm4, %xmm3
440; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
441; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,0,65535,65535,65535,65535,65535]
442; SSE2-NEXT:    pand %xmm2, %xmm1
443; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
444; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,3,1,3,4,5,6,7]
445; SSE2-NEXT:    pandn %xmm0, %xmm2
446; SSE2-NEXT:    por %xmm1, %xmm2
447; SSE2-NEXT:    movq %xmm2, 16(%rdi)
448; SSE2-NEXT:    movdqa %xmm3, (%rdi)
449; SSE2-NEXT:    retq
450;
451; SSE42-LABEL: v12i16:
452; SSE42:       # BB#0:
453; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
454; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
455; SSE42-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
456; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0],xmm3[1,2],xmm2[3,4,5,6,7]
457; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,3]
458; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,2,3,10,11,10,11,4,5,12,13]
459; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
460; SSE42-NEXT:    movdqa %xmm0, (%rdi)
461; SSE42-NEXT:    movq %xmm3, 16(%rdi)
462; SSE42-NEXT:    retq
463;
464; AVX1-LABEL: v12i16:
465; AVX1:       # BB#0:
466; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
467; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
468; AVX1-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
469; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2],xmm2[3],xmm3[4,5,6,7]
470; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,3]
471; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,2,3,10,11,10,11,4,5,12,13]
472; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
473; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
474; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
475; AVX1-NEXT:    retq
476;
477; AVX2-LABEL: v12i16:
478; AVX2:       # BB#0:
479; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
480; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
481; AVX2-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
482; AVX2-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2],xmm2[3],xmm3[4,5,6,7]
483; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
484; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,8,9,2,3,10,11,10,11,4,5,12,13]
485; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
486; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
487; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
488; AVX2-NEXT:    retq
489  %r = shufflevector <8 x i16> %a, <8 x i16> %b, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
490  store <12 x i16> %r, <12 x i16>* %p
491  ret void
492}
493
494define void @v12i32(<8 x i32> %a, <8 x i32> %b, <12 x i32>* %p) nounwind {
495; SSE2-LABEL: v12i32:
496; SSE2:       # BB#0:
497; SSE2-NEXT:    movdqa %xmm0, %xmm3
498; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
499; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,2,2]
500; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,1,0,1]
501; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm3[3,0]
502; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0,2]
503; SSE2-NEXT:    movdqa %xmm2, %xmm4
504; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,0],xmm1[1,0]
505; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm1[2,2]
506; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
507; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm4[3,0]
508; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm0[0,2]
509; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,3,2,2]
510; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
511; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0]
512; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,2]
513; SSE2-NEXT:    movaps %xmm0, 32(%rdi)
514; SSE2-NEXT:    movaps %xmm4, 16(%rdi)
515; SSE2-NEXT:    movaps %xmm3, (%rdi)
516; SSE2-NEXT:    retq
517;
518; SSE42-LABEL: v12i32:
519; SSE42:       # BB#0:
520; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,1,1]
521; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,1,0,1]
522; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5,6,7]
523; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]
524; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5],xmm4[6,7]
525; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,2,2]
526; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3],xmm4[4,5,6,7]
527; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm0[4,5],xmm4[6,7]
528; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
529; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
530; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3],xmm2[4,5,6,7]
531; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,2,3,3]
532; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7]
533; SSE42-NEXT:    movdqa %xmm0, 32(%rdi)
534; SSE42-NEXT:    movdqa %xmm4, 16(%rdi)
535; SSE42-NEXT:    movdqa %xmm3, (%rdi)
536; SSE42-NEXT:    retq
537;
538; AVX1-LABEL: v12i32:
539; AVX1:       # BB#0:
540; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,0,1]
541; AVX1-NEXT:    vmovsldup {{.*#+}} ymm2 = ymm2[0,0,2,2,4,4,6,6]
542; AVX1-NEXT:    vpermilps {{.*#+}} ymm3 = ymm0[0,u,u,1,5,u,u,6]
543; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3,4,5],ymm2[6],ymm3[7]
544; AVX1-NEXT:    vmovddup {{.*#+}} xmm3 = xmm1[0,0]
545; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm3
546; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
547; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
548; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,3],xmm3[3,3]
549; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,1]
550; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
551; AVX1-NEXT:    vmovaps %xmm0, 32(%rdi)
552; AVX1-NEXT:    vmovaps %ymm2, (%rdi)
553; AVX1-NEXT:    vzeroupper
554; AVX1-NEXT:    retq
555;
556; AVX2-LABEL: v12i32:
557; AVX2:       # BB#0:
558; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
559; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[3,3,2,3,7,7,6,7]
560; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
561; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1,2],xmm2[3]
562; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = <0,4,u,1,5,u,2,6>
563; AVX2-NEXT:    vpermd %ymm0, %ymm3, %ymm0
564; AVX2-NEXT:    vpbroadcastq %xmm1, %ymm1
565; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
566; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
567; AVX2-NEXT:    vmovdqa %xmm2, 32(%rdi)
568; AVX2-NEXT:    vzeroupper
569; AVX2-NEXT:    retq
570  %r = shufflevector <8 x i32> %a, <8 x i32> %b, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
571  store <12 x i32> %r, <12 x i32>* %p
572  ret void
573}
574
575define void @pr29025(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <12 x i8> *%p) nounwind {
576; SSE2-LABEL: pr29025:
577; SSE2:       # BB#0:
578; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255]
579; SSE2-NEXT:    pand %xmm3, %xmm1
580; SSE2-NEXT:    pand %xmm3, %xmm0
581; SSE2-NEXT:    packuswb %xmm1, %xmm0
582; SSE2-NEXT:    packuswb %xmm0, %xmm0
583; SSE2-NEXT:    pxor %xmm1, %xmm1
584; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
585; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
586; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,3,1,3,4,5,6,7]
587; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
588; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
589; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,1,4,5,6,7]
590; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,4]
591; SSE2-NEXT:    packuswb %xmm1, %xmm0
592; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,255,255]
593; SSE2-NEXT:    pand %xmm1, %xmm0
594; SSE2-NEXT:    pand %xmm3, %xmm2
595; SSE2-NEXT:    packuswb %xmm2, %xmm2
596; SSE2-NEXT:    packuswb %xmm2, %xmm2
597; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,1,1,4,5,6,7]
598; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
599; SSE2-NEXT:    pandn %xmm2, %xmm1
600; SSE2-NEXT:    por %xmm0, %xmm1
601; SSE2-NEXT:    movq %xmm1, (%rdi)
602; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
603; SSE2-NEXT:    movd %xmm0, 8(%rdi)
604; SSE2-NEXT:    retq
605;
606; SSE42-LABEL: pr29025:
607; SSE42:       # BB#0:
608; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
609; SSE42-NEXT:    pshufb %xmm3, %xmm1
610; SSE42-NEXT:    pshufb %xmm3, %xmm0
611; SSE42-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
612; SSE42-NEXT:    pshufb %xmm3, %xmm2
613; SSE42-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
614; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8,1,2,10,3,4,12,5,6,14,7,u,u,u,u]
615; SSE42-NEXT:    pextrd $2, %xmm0, 8(%rdi)
616; SSE42-NEXT:    movq %xmm0, (%rdi)
617; SSE42-NEXT:    retq
618;
619; AVX-LABEL: pr29025:
620; AVX:       # BB#0:
621; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
622; AVX-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
623; AVX-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
624; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
625; AVX-NEXT:    vpshufb %xmm3, %xmm2, %xmm1
626; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
627; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,1,2,10,3,4,12,5,6,14,7,u,u,u,u]
628; AVX-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
629; AVX-NEXT:    vmovq %xmm0, (%rdi)
630; AVX-NEXT:    retq
631  %s1 = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
632  %s2 = shufflevector <4 x i8> %c, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
633  %r = shufflevector <8 x i8> %s1, <8 x i8> %s2, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
634  store <12 x i8> %r, <12 x i8>* %p, align 1
635  ret void
636}
637
638define void @interleave_24i8_out(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8 x i8>* %q3) nounwind {
639; SSE2-LABEL: interleave_24i8_out:
640; SSE2:       # BB#0:
641; SSE2-NEXT:    movdqu (%rdi), %xmm0
642; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
643; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,0,255,255,0,255,255,255,255,255,255,255,255,255,255]
644; SSE2-NEXT:    movdqa %xmm0, %xmm2
645; SSE2-NEXT:    pand %xmm3, %xmm2
646; SSE2-NEXT:    pandn %xmm1, %xmm3
647; SSE2-NEXT:    por %xmm2, %xmm3
648; SSE2-NEXT:    pxor %xmm2, %xmm2
649; SSE2-NEXT:    movdqa %xmm3, %xmm4
650; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
651; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [65535,0,65535,65535,0,65535,65535,0]
652; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
653; SSE2-NEXT:    pand %xmm5, %xmm3
654; SSE2-NEXT:    pandn %xmm4, %xmm5
655; SSE2-NEXT:    por %xmm3, %xmm5
656; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,1,3]
657; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]
658; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,3,2,1]
659; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,3,2,1,4,5,6,7]
660; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]
661; SSE2-NEXT:    packuswb %xmm0, %xmm3
662; SSE2-NEXT:    movq %xmm3, (%rsi)
663; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [0,255,255,0,255,255,0,255,255,255,255,255,255,255,255,255]
664; SSE2-NEXT:    movdqa %xmm0, %xmm4
665; SSE2-NEXT:    pand %xmm3, %xmm4
666; SSE2-NEXT:    pandn %xmm1, %xmm3
667; SSE2-NEXT:    por %xmm4, %xmm3
668; SSE2-NEXT:    movdqa %xmm3, %xmm4
669; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
670; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [65535,65535,0,65535,65535,0,65535,65535]
671; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
672; SSE2-NEXT:    pand %xmm5, %xmm3
673; SSE2-NEXT:    pandn %xmm4, %xmm5
674; SSE2-NEXT:    por %xmm3, %xmm5
675; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[2,1,0,3,4,5,6,7]
676; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]
677; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,3,2,1]
678; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[1,2,3,0,4,5,6,7]
679; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
680; SSE2-NEXT:    packuswb %xmm0, %xmm3
681; SSE2-NEXT:    movq %xmm3, (%rdx)
682; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,0,255,255,0,255,255,0,255,255,255,255,255,255,255,255]
683; SSE2-NEXT:    pand %xmm3, %xmm0
684; SSE2-NEXT:    pandn %xmm1, %xmm3
685; SSE2-NEXT:    por %xmm0, %xmm3
686; SSE2-NEXT:    movdqa %xmm3, %xmm0
687; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
688; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,0,65535,65535,0,65535]
689; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
690; SSE2-NEXT:    pand %xmm1, %xmm3
691; SSE2-NEXT:    pandn %xmm0, %xmm1
692; SSE2-NEXT:    por %xmm3, %xmm1
693; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,0]
694; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]
695; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0]
696; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]
697; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
698; SSE2-NEXT:    packuswb %xmm0, %xmm0
699; SSE2-NEXT:    movq %xmm0, (%rcx)
700; SSE2-NEXT:    retq
701;
702; SSE42-LABEL: interleave_24i8_out:
703; SSE42:       # BB#0:
704; SSE42-NEXT:    movdqu (%rdi), %xmm0
705; SSE42-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
706; SSE42-NEXT:    movdqa %xmm1, %xmm2
707; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm2[2,5,u,u,u,u,u,u,u,u]
708; SSE42-NEXT:    movdqa %xmm0, %xmm3
709; SSE42-NEXT:    pshufb {{.*#+}} xmm3 = xmm3[0,3,6,9,12,15],zero,zero,xmm3[u,u,u,u,u,u,u,u]
710; SSE42-NEXT:    por %xmm2, %xmm3
711; SSE42-NEXT:    movq %xmm3, (%rsi)
712; SSE42-NEXT:    movdqa %xmm1, %xmm2
713; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm2[0,3,6,u,u,u,u,u,u,u,u]
714; SSE42-NEXT:    movdqa %xmm0, %xmm3
715; SSE42-NEXT:    pshufb {{.*#+}} xmm3 = xmm3[1,4,7,10,13],zero,zero,zero,xmm3[u,u,u,u,u,u,u,u]
716; SSE42-NEXT:    por %xmm2, %xmm3
717; SSE42-NEXT:    movq %xmm3, (%rdx)
718; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]
719; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
720; SSE42-NEXT:    por %xmm1, %xmm0
721; SSE42-NEXT:    movq %xmm0, (%rcx)
722; SSE42-NEXT:    retq
723;
724; AVX-LABEL: interleave_24i8_out:
725; AVX:       # BB#0:
726; AVX-NEXT:    vmovdqu (%rdi), %xmm0
727; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
728; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]
729; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]
730; AVX-NEXT:    vpor %xmm2, %xmm3, %xmm2
731; AVX-NEXT:    vmovq %xmm2, (%rsi)
732; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]
733; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
734; AVX-NEXT:    vpor %xmm2, %xmm3, %xmm2
735; AVX-NEXT:    vmovq %xmm2, (%rdx)
736; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]
737; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
738; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
739; AVX-NEXT:    vmovq %xmm0, (%rcx)
740; AVX-NEXT:    retq
741  %wide.vec = load <24 x i8>, <24 x i8>* %p, align 4
742  %s1 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
743  %s2 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
744  %s3 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
745  store <8 x i8> %s1, <8 x i8>* %q1, align 4
746  store <8 x i8> %s2, <8 x i8>* %q2, align 4
747  store <8 x i8> %s3, <8 x i8>* %q3, align 4
748  ret void
749}
750
751define void @interleave_24i8_in(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8 x i8>* %q3) nounwind {
752; SSE2-LABEL: interleave_24i8_in:
753; SSE2:       # BB#0:
754; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
755; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
756; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
757; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
758; SSE2-NEXT:    pxor %xmm2, %xmm2
759; SSE2-NEXT:    movdqa %xmm1, %xmm3
760; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
761; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,2,2]
762; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [65535,65535,0,65535,65535,0,65535,65535]
763; SSE2-NEXT:    pand %xmm5, %xmm4
764; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
765; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[0,1,3,3,4,5,6,7]
766; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,6,7]
767; SSE2-NEXT:    pandn %xmm2, %xmm5
768; SSE2-NEXT:    por %xmm4, %xmm5
769; SSE2-NEXT:    movdqa %xmm3, %xmm2
770; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
771; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]
772; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,2,4,5,6,7]
773; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,4,5]
774; SSE2-NEXT:    packuswb %xmm5, %xmm2
775; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,0,255]
776; SSE2-NEXT:    pand %xmm4, %xmm2
777; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,1,0,1]
778; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[0,0,0,3,4,5,6,7]
779; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,6,6]
780; SSE2-NEXT:    pandn %xmm5, %xmm4
781; SSE2-NEXT:    por %xmm2, %xmm4
782; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
783; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
784; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[2,1,1,0,4,5,6,7]
785; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,7]
786; SSE2-NEXT:    packuswb %xmm0, %xmm1
787; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,255,0,255,255,0,255,255,255,255,255,255,255,255]
788; SSE2-NEXT:    pand %xmm2, %xmm1
789; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,1,3,3,4,5,6,7]
790; SSE2-NEXT:    pandn %xmm0, %xmm2
791; SSE2-NEXT:    por %xmm1, %xmm2
792; SSE2-NEXT:    movq %xmm2, 16(%rdi)
793; SSE2-NEXT:    movdqu %xmm4, (%rdi)
794; SSE2-NEXT:    retq
795;
796; SSE42-LABEL: interleave_24i8_in:
797; SSE42:       # BB#0:
798; SSE42-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
799; SSE42-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
800; SSE42-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
801; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
802; SSE42-NEXT:    movdqa %xmm0, %xmm1
803; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,8],zero,xmm1[1,9],zero,xmm1[2,10],zero,xmm1[3,11],zero,xmm1[4,12],zero,xmm1[5]
804; SSE42-NEXT:    movdqa %xmm2, %xmm3
805; SSE42-NEXT:    pshufb {{.*#+}} xmm3 = zero,zero,xmm3[0],zero,zero,xmm3[1],zero,zero,xmm3[2],zero,zero,xmm3[3],zero,zero,xmm3[4],zero
806; SSE42-NEXT:    por %xmm1, %xmm3
807; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[13],zero,xmm0[6,14],zero,xmm0[7,15],zero,xmm0[u,u,u,u,u,u,u,u]
808; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = zero,xmm2[5],zero,zero,xmm2[6],zero,zero,xmm2[7,u,u,u,u,u,u,u,u]
809; SSE42-NEXT:    por %xmm0, %xmm2
810; SSE42-NEXT:    movq %xmm2, 16(%rdi)
811; SSE42-NEXT:    movdqu %xmm3, (%rdi)
812; SSE42-NEXT:    retq
813;
814; AVX-LABEL: interleave_24i8_in:
815; AVX:       # BB#0:
816; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
817; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
818; AVX-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
819; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
820; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[0,8],zero,xmm0[1,9],zero,xmm0[2,10],zero,xmm0[3,11],zero,xmm0[4,12],zero,xmm0[5]
821; AVX-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm2[0],zero,zero,xmm2[1],zero,zero,xmm2[2],zero,zero,xmm2[3],zero,zero,xmm2[4],zero
822; AVX-NEXT:    vpor %xmm3, %xmm1, %xmm1
823; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[13],zero,xmm0[6,14],zero,xmm0[7,15],zero,xmm0[u,u,u,u,u,u,u,u]
824; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[5],zero,zero,xmm2[6],zero,zero,xmm2[7,u,u,u,u,u,u,u,u]
825; AVX-NEXT:    vpor %xmm2, %xmm0, %xmm0
826; AVX-NEXT:    vmovq %xmm0, 16(%rdi)
827; AVX-NEXT:    vmovdqu %xmm1, (%rdi)
828; AVX-NEXT:    retq
829  %s1 = load <8 x i8>, <8 x i8>* %q1, align 4
830  %s2 = load <8 x i8>, <8 x i8>* %q2, align 4
831  %s3 = load <8 x i8>, <8 x i8>* %q3, align 4
832  %t1 = shufflevector <8 x i8> %s1, <8 x i8> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
833  %t2 = shufflevector <8 x i8> %s3, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
834  %interleaved = shufflevector <16 x i8> %t1, <16 x i8> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
835  store <24 x i8> %interleaved, <24 x i8>* %p, align 4
836  ret void
837}
838
839
840define void @interleave_24i16_out(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2, <8 x i16>* %q3) nounwind {
841; SSE2-LABEL: interleave_24i16_out:
842; SSE2:       # BB#0:
843; SSE2-NEXT:    movdqu (%rdi), %xmm3
844; SSE2-NEXT:    movdqu 16(%rdi), %xmm2
845; SSE2-NEXT:    movdqu 32(%rdi), %xmm8
846; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,65535,65535,0,65535,65535,0]
847; SSE2-NEXT:    movdqa %xmm3, %xmm4
848; SSE2-NEXT:    pand %xmm1, %xmm4
849; SSE2-NEXT:    pandn %xmm2, %xmm1
850; SSE2-NEXT:    por %xmm4, %xmm1
851; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
852; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]
853; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
854; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7]
855; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,7]
856; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[0,1,2,1]
857; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
858; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,0],xmm1[2,0]
859; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,0]
860; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [65535,65535,0,65535,65535,0,65535,65535]
861; SSE2-NEXT:    movdqa %xmm4, %xmm5
862; SSE2-NEXT:    pandn %xmm2, %xmm5
863; SSE2-NEXT:    movdqa %xmm3, %xmm6
864; SSE2-NEXT:    pand %xmm4, %xmm6
865; SSE2-NEXT:    por %xmm5, %xmm6
866; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm6[2,1,2,3,4,5,6,7]
867; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,4,7]
868; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3]
869; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[1,2,3,0,4,5,6,7]
870; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,6,7]
871; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [65535,65535,65535,65535,65535,0,0,0]
872; SSE2-NEXT:    pand %xmm6, %xmm5
873; SSE2-NEXT:    pshuflw {{.*#+}} xmm7 = xmm8[0,3,2,3,4,5,6,7]
874; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[0,1,0,3]
875; SSE2-NEXT:    pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,4,5,6]
876; SSE2-NEXT:    movdqa %xmm6, %xmm0
877; SSE2-NEXT:    pandn %xmm7, %xmm0
878; SSE2-NEXT:    por %xmm5, %xmm0
879; SSE2-NEXT:    pand %xmm4, %xmm2
880; SSE2-NEXT:    pandn %xmm3, %xmm4
881; SSE2-NEXT:    por %xmm2, %xmm4
882; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[3,1,2,0]
883; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]
884; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,1,0,3]
885; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[2,1,0,3,4,5,6,7]
886; SSE2-NEXT:    pand %xmm6, %xmm2
887; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm8[0,1,2,3,4,7,6,7]
888; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0]
889; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,4,5]
890; SSE2-NEXT:    pandn %xmm3, %xmm6
891; SSE2-NEXT:    por %xmm2, %xmm6
892; SSE2-NEXT:    movups %xmm1, (%rsi)
893; SSE2-NEXT:    movdqu %xmm0, (%rdx)
894; SSE2-NEXT:    movdqu %xmm6, (%rcx)
895; SSE2-NEXT:    retq
896;
897; SSE42-LABEL: interleave_24i16_out:
898; SSE42:       # BB#0:
899; SSE42-NEXT:    movdqu (%rdi), %xmm0
900; SSE42-NEXT:    movdqu 16(%rdi), %xmm1
901; SSE42-NEXT:    movdqu 32(%rdi), %xmm2
902; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,1,2,1]
903; SSE42-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6,5]
904; SSE42-NEXT:    movdqa %xmm0, %xmm4
905; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0],xmm1[1],xmm4[2,3],xmm1[4],xmm4[5,6],xmm1[7]
906; SSE42-NEXT:    pshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
907; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5],xmm3[6,7]
908; SSE42-NEXT:    movdqa %xmm2, %xmm3
909; SSE42-NEXT:    pshufb {{.*#+}} xmm3 = xmm3[0,1,6,7,4,5,6,7,0,1,0,1,6,7,12,13]
910; SSE42-NEXT:    movdqa %xmm0, %xmm5
911; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm5[0,1],xmm1[2],xmm5[3,4],xmm1[5],xmm5[6,7]
912; SSE42-NEXT:    pshufb {{.*#+}} xmm5 = xmm5[2,3,8,9,14,15,4,5,10,11,10,11,8,9,14,15]
913; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm3[5,6,7]
914; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6,7,8,9,2,3,8,9,14,15]
915; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
916; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
917; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm2[5,6,7]
918; SSE42-NEXT:    movdqu %xmm4, (%rsi)
919; SSE42-NEXT:    movdqu %xmm5, (%rdx)
920; SSE42-NEXT:    movdqu %xmm1, (%rcx)
921; SSE42-NEXT:    retq
922;
923; AVX1-LABEL: interleave_24i16_out:
924; AVX1:       # BB#0:
925; AVX1-NEXT:    vmovdqu 32(%rdi), %xmm0
926; AVX1-NEXT:    vmovups (%rdi), %ymm1
927; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
928; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5,6],xmm2[7]
929; AVX1-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
930; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[0,1,2,1]
931; AVX1-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
932; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm4[6,7]
933; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
934; AVX1-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11,10,11,8,9,14,15]
935; AVX1-NEXT:    vpshufb {{.*#+}} xmm5 = xmm0[0,1,6,7,4,5,6,7,0,1,0,1,6,7,12,13]
936; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm5[5,6,7]
937; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
938; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
939; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,7,8,9,2,3,8,9,14,15]
940; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4],xmm0[5,6,7]
941; AVX1-NEXT:    vmovdqu %xmm3, (%rsi)
942; AVX1-NEXT:    vmovdqu %xmm4, (%rdx)
943; AVX1-NEXT:    vmovdqu %xmm0, (%rcx)
944; AVX1-NEXT:    vzeroupper
945; AVX1-NEXT:    retq
946;
947; AVX2-LABEL: interleave_24i16_out:
948; AVX2:       # BB#0:
949; AVX2-NEXT:    vmovdqu (%rdi), %ymm0
950; AVX2-NEXT:    vmovdqu 32(%rdi), %xmm1
951; AVX2-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
952; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
953; AVX2-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6],xmm3[7]
954; AVX2-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11]
955; AVX2-NEXT:    vpblendw {{.*#+}} ymm3 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]
956; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
957; AVX2-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]
958; AVX2-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]
959; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7],ymm0[8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14],ymm1[15]
960; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
961; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
962; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]
963; AVX2-NEXT:    vmovdqu %xmm2, (%rsi)
964; AVX2-NEXT:    vmovdqu %xmm3, (%rdx)
965; AVX2-NEXT:    vmovdqu %xmm0, (%rcx)
966; AVX2-NEXT:    vzeroupper
967; AVX2-NEXT:    retq
968  %wide.vec = load <24 x i16>, <24 x i16>* %p, align 4
969  %s1 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
970  %s2 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
971  %s3 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
972  store <8 x i16> %s1, <8 x i16>* %q1, align 4
973  store <8 x i16> %s2, <8 x i16>* %q2, align 4
974  store <8 x i16> %s3, <8 x i16>* %q3, align 4
975  ret void
976}
977
978define void @interleave_24i16_in(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2, <8 x i16>* %q3) nounwind {
979; SSE2-LABEL: interleave_24i16_in:
980; SSE2:       # BB#0:
981; SSE2-NEXT:    movdqu (%rsi), %xmm3
982; SSE2-NEXT:    movdqu (%rdx), %xmm2
983; SSE2-NEXT:    movdqu (%rcx), %xmm1
984; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,0,3]
985; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [65535,65535,0,65535,65535,0,65535,65535]
986; SSE2-NEXT:    movdqa %xmm0, %xmm5
987; SSE2-NEXT:    pandn %xmm4, %xmm5
988; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,3,3,3]
989; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,1,2,2]
990; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
991; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]
992; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,1,2,2,4,5,6,7]
993; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,4,5]
994; SSE2-NEXT:    pand %xmm0, %xmm3
995; SSE2-NEXT:    por %xmm5, %xmm3
996; SSE2-NEXT:    movdqa %xmm0, %xmm5
997; SSE2-NEXT:    pandn %xmm4, %xmm5
998; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,1,3,3,4,5,6,7]
999; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
1000; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,1,2,3]
1001; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[2,3,2,0,4,5,6,7]
1002; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,7]
1003; SSE2-NEXT:    pand %xmm0, %xmm2
1004; SSE2-NEXT:    por %xmm5, %xmm2
1005; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [65535,0,65535,65535,0,65535,65535,0]
1006; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,2]
1007; SSE2-NEXT:    pand %xmm5, %xmm1
1008; SSE2-NEXT:    pandn %xmm6, %xmm5
1009; SSE2-NEXT:    por %xmm1, %xmm5
1010; SSE2-NEXT:    pand %xmm0, %xmm5
1011; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm4[0,1,2,3,4,4,6,7]
1012; SSE2-NEXT:    pandn %xmm1, %xmm0
1013; SSE2-NEXT:    por %xmm5, %xmm0
1014; SSE2-NEXT:    movdqu %xmm0, 16(%rdi)
1015; SSE2-NEXT:    movdqu %xmm2, 32(%rdi)
1016; SSE2-NEXT:    movdqu %xmm3, (%rdi)
1017; SSE2-NEXT:    retq
1018;
1019; SSE42-LABEL: interleave_24i16_in:
1020; SSE42:       # BB#0:
1021; SSE42-NEXT:    movdqu (%rsi), %xmm0
1022; SSE42-NEXT:    movdqu (%rdx), %xmm1
1023; SSE42-NEXT:    movdqu (%rcx), %xmm2
1024; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,2]
1025; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,3,3,3]
1026; SSE42-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1027; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
1028; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,0,3]
1029; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm0[0,1],xmm5[2],xmm0[3,4],xmm5[5],xmm0[6,7]
1030; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,2]
1031; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6],xmm3[7]
1032; SSE42-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[0,1,3,3,4,5,6,7]
1033; SSE42-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,6,7]
1034; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm0[0,1],xmm3[2],xmm0[3,4],xmm3[5],xmm0[6,7]
1035; SSE42-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1036; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[4,5,6,7,4,5,8,9,10,11,10,11,12,13,14,15]
1037; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm1[0,1],xmm4[2],xmm1[3,4],xmm4[5],xmm1[6,7]
1038; SSE42-NEXT:    movdqu %xmm4, 32(%rdi)
1039; SSE42-NEXT:    movdqu %xmm3, 16(%rdi)
1040; SSE42-NEXT:    movdqu %xmm5, (%rdi)
1041; SSE42-NEXT:    retq
1042;
1043; AVX1-LABEL: interleave_24i16_in:
1044; AVX1:       # BB#0:
1045; AVX1-NEXT:    vmovdqu (%rsi), %xmm0
1046; AVX1-NEXT:    vmovdqu (%rdx), %xmm1
1047; AVX1-NEXT:    vmovdqu (%rcx), %xmm2
1048; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,2]
1049; AVX1-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm1[0,1,3,3,4,5,6,7]
1050; AVX1-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,4,6,7]
1051; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]
1052; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[1,1,2,2]
1053; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1,2],xmm4[3],xmm3[4,5],xmm4[6],xmm3[7]
1054; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1055; AVX1-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
1056; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[0,0,0,3]
1057; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
1058; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
1059; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1060; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]
1061; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]
1062; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
1063; AVX1-NEXT:    vmovdqu %xmm0, 32(%rdi)
1064; AVX1-NEXT:    vmovups %ymm3, (%rdi)
1065; AVX1-NEXT:    vzeroupper
1066; AVX1-NEXT:    retq
1067;
1068; AVX2-LABEL: interleave_24i16_in:
1069; AVX2:       # BB#0:
1070; AVX2-NEXT:    vmovdqu (%rsi), %xmm0
1071; AVX2-NEXT:    vmovdqu (%rdx), %xmm1
1072; AVX2-NEXT:    vmovdqu (%rcx), %xmm2
1073; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm3
1074; AVX2-NEXT:    vpshufb {{.*#+}} ymm4 = ymm3[0,1,2,3,6,7,2,3,8,9,8,9,4,5,6,7,16,17,18,19,22,23,18,19,24,25,24,25,20,21,22,23]
1075; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm3[2,3,0,1]
1076; AVX2-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,0,1,u,u,u,u,2,3,u,u,u,u,4,5,u,u,22,23,u,u,u,u,24,25,u,u,u,u,26,27]
1077; AVX2-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7],ymm4[8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13,14],ymm3[15]
1078; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = <u,0,0,u,1,1,u,2>
1079; AVX2-NEXT:    vpermd %ymm2, %ymm4, %ymm4
1080; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255]
1081; AVX2-NEXT:    vpblendvb %ymm5, %ymm3, %ymm4, %ymm3
1082; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1083; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]
1084; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]
1085; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
1086; AVX2-NEXT:    vmovdqu %xmm0, 32(%rdi)
1087; AVX2-NEXT:    vmovdqu %ymm3, (%rdi)
1088; AVX2-NEXT:    vzeroupper
1089; AVX2-NEXT:    retq
1090  %s1 = load <8 x i16>, <8 x i16>* %q1, align 4
1091  %s2 = load <8 x i16>, <8 x i16>* %q2, align 4
1092  %s3 = load <8 x i16>, <8 x i16>* %q3, align 4
1093  %t1 = shufflevector <8 x i16> %s1, <8 x i16> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
1094  %t2 = shufflevector <8 x i16> %s3, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1095  %interleaved = shufflevector <16 x i16> %t1, <16 x i16> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
1096  store <24 x i16> %interleaved, <24 x i16>* %p, align 4
1097  ret void
1098}
1099
1100define void @interleave_24i32_out(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2, <8 x i32>* %q3) nounwind {
1101; SSE2-LABEL: interleave_24i32_out:
1102; SSE2:       # BB#0:
1103; SSE2-NEXT:    movups 80(%rdi), %xmm5
1104; SSE2-NEXT:    movups 64(%rdi), %xmm8
1105; SSE2-NEXT:    movups (%rdi), %xmm0
1106; SSE2-NEXT:    movups 16(%rdi), %xmm6
1107; SSE2-NEXT:    movups 32(%rdi), %xmm2
1108; SSE2-NEXT:    movups 48(%rdi), %xmm1
1109; SSE2-NEXT:    movaps %xmm1, %xmm3
1110; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,3],xmm8[2,3]
1111; SSE2-NEXT:    movaps %xmm5, %xmm4
1112; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,0],xmm3[2,0]
1113; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,0]
1114; SSE2-NEXT:    movaps %xmm0, %xmm4
1115; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,3],xmm6[2,3]
1116; SSE2-NEXT:    movaps %xmm2, %xmm7
1117; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[1,0],xmm4[2,0]
1118; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm7[2,0]
1119; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[2,3,0,1]
1120; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm6[0,0]
1121; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm6[3,3]
1122; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm2[0,1,0,3]
1123; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,0]
1124; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
1125; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
1126; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm8[0,0]
1127; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm8[3,3]
1128; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,1,0,3]
1129; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm1[2,0]
1130; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm5[2,0]
1131; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,2,3]
1132; SSE2-NEXT:    punpckldq {{.*#+}} xmm10 = xmm10[0],xmm5[0],xmm10[1],xmm5[1]
1133; SSE2-NEXT:    movsd {{.*#+}} xmm9 = xmm10[0],xmm9[1]
1134; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm8[1,1,2,3]
1135; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1136; SSE2-NEXT:    movsd {{.*#+}} xmm7 = xmm2[0],xmm7[1]
1137; SSE2-NEXT:    movups %xmm3, 16(%rsi)
1138; SSE2-NEXT:    movups %xmm4, (%rsi)
1139; SSE2-NEXT:    movups %xmm1, 16(%rdx)
1140; SSE2-NEXT:    movups %xmm0, (%rdx)
1141; SSE2-NEXT:    movupd %xmm7, 16(%rcx)
1142; SSE2-NEXT:    movupd %xmm9, (%rcx)
1143; SSE2-NEXT:    retq
1144;
1145; SSE42-LABEL: interleave_24i32_out:
1146; SSE42:       # BB#0:
1147; SSE42-NEXT:    movdqu 80(%rdi), %xmm9
1148; SSE42-NEXT:    movdqu 64(%rdi), %xmm10
1149; SSE42-NEXT:    movdqu (%rdi), %xmm4
1150; SSE42-NEXT:    movdqu 16(%rdi), %xmm2
1151; SSE42-NEXT:    movdqu 32(%rdi), %xmm11
1152; SSE42-NEXT:    movdqu 48(%rdi), %xmm5
1153; SSE42-NEXT:    pshufd {{.*#+}} xmm8 = xmm11[0,1,0,1]
1154; SSE42-NEXT:    movdqa %xmm2, %xmm7
1155; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm4[2,3],xmm7[4,5,6,7]
1156; SSE42-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[2,3,0,1]
1157; SSE42-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,3],xmm2[2,3]
1158; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5],xmm8[6,7]
1159; SSE42-NEXT:    movdqa %xmm10, %xmm1
1160; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm5[2,3],xmm1[4,5,6,7]
1161; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[2,3,0,1]
1162; SSE42-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm10[2,3]
1163; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm9[0,1,0,1]
1164; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm5[0,1,2,3,4,5],xmm3[6,7]
1165; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm11[0,1,2,2]
1166; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,0,3,3]
1167; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5],xmm5[6,7]
1168; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,0,3,3]
1169; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm9[0,1,2,2]
1170; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm1[0,1,2,3,4,5],xmm5[6,7]
1171; SSE42-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1],xmm2[2,3],xmm6[4,5,6,7]
1172; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[0,1,0,3]
1173; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm6[0,1,2,3],xmm1[4,5,6,7]
1174; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm10[2,3],xmm0[4,5,6,7]
1175; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm9[0,1,0,3]
1176; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1,2,3],xmm2[4,5,6,7]
1177; SSE42-NEXT:    movdqu %xmm3, 16(%rsi)
1178; SSE42-NEXT:    movdqu %xmm4, (%rsi)
1179; SSE42-NEXT:    movdqu %xmm5, 16(%rdx)
1180; SSE42-NEXT:    movdqu %xmm7, (%rdx)
1181; SSE42-NEXT:    movdqu %xmm2, 16(%rcx)
1182; SSE42-NEXT:    movdqu %xmm1, (%rcx)
1183; SSE42-NEXT:    retq
1184;
1185; AVX1-LABEL: interleave_24i32_out:
1186; AVX1:       # BB#0:
1187; AVX1-NEXT:    vmovups (%rdi), %ymm0
1188; AVX1-NEXT:    vmovups 32(%rdi), %ymm1
1189; AVX1-NEXT:    vmovups 64(%rdi), %ymm2
1190; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
1191; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = zero,zero,xmm2[2],xmm3[1]
1192; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm4
1193; AVX1-NEXT:    vblendps {{.*#+}} ymm5 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
1194; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm6
1195; AVX1-NEXT:    vblendps {{.*#+}} xmm5 = xmm5[0,1],xmm6[2],xmm5[3]
1196; AVX1-NEXT:    vpermilps {{.*#+}} xmm5 = xmm5[0,3,2,1]
1197; AVX1-NEXT:    vpermilps {{.*#+}} xmm6 = xmm6[0,3,2,3]
1198; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm5
1199; AVX1-NEXT:    vblendpd {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3]
1200; AVX1-NEXT:    vblendps {{.*#+}} xmm5 = xmm2[0,1],xmm3[2],xmm2[3]
1201; AVX1-NEXT:    vpermilps {{.*#+}} xmm5 = xmm5[0,0,3,2]
1202; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm5
1203; AVX1-NEXT:    vblendps {{.*#+}} ymm6 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1204; AVX1-NEXT:    vextractf128 $1, %ymm6, %xmm7
1205; AVX1-NEXT:    vblendps {{.*#+}} xmm6 = xmm7[0],xmm6[1,2],xmm7[3]
1206; AVX1-NEXT:    vpermilps {{.*#+}} xmm6 = xmm6[1,0,3,2]
1207; AVX1-NEXT:    vmovshdup {{.*#+}} xmm7 = xmm7[1,1,3,3]
1208; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm6, %ymm6
1209; AVX1-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4],ymm5[5,6,7]
1210; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0,3]
1211; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm2
1212; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
1213; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
1214; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
1215; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,1,0,3]
1216; AVX1-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
1217; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
1218; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
1219; AVX1-NEXT:    vmovupd %ymm4, (%rsi)
1220; AVX1-NEXT:    vmovups %ymm5, (%rdx)
1221; AVX1-NEXT:    vmovups %ymm0, (%rcx)
1222; AVX1-NEXT:    vzeroupper
1223; AVX1-NEXT:    retq
1224;
1225; AVX2-LABEL: interleave_24i32_out:
1226; AVX2:       # BB#0:
1227; AVX2-NEXT:    vmovdqu (%rdi), %ymm0
1228; AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
1229; AVX2-NEXT:    vmovdqu 64(%rdi), %ymm2
1230; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = <u,u,u,u,u,u,2,5>
1231; AVX2-NEXT:    vpermd %ymm2, %ymm3, %ymm3
1232; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
1233; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = <0,3,6,1,4,7,u,u>
1234; AVX2-NEXT:    vpermd %ymm4, %ymm5, %ymm4
1235; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
1236; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = <u,u,u,u,u,0,3,6>
1237; AVX2-NEXT:    vpermd %ymm2, %ymm4, %ymm4
1238; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1239; AVX2-NEXT:    vmovdqa {{.*#+}} ymm6 = <1,4,7,2,5,u,u,u>
1240; AVX2-NEXT:    vpermd %ymm5, %ymm6, %ymm5
1241; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5,6,7]
1242; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
1243; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = <2,5,0,3,6,u,u,u>
1244; AVX2-NEXT:    vpermd %ymm0, %ymm1, %ymm0
1245; AVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm2[0,1,0,3,4,5,4,7]
1246; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,3]
1247; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
1248; AVX2-NEXT:    vmovdqu %ymm3, (%rsi)
1249; AVX2-NEXT:    vmovdqu %ymm4, (%rdx)
1250; AVX2-NEXT:    vmovdqu %ymm0, (%rcx)
1251; AVX2-NEXT:    vzeroupper
1252; AVX2-NEXT:    retq
1253  %wide.vec = load <24 x i32>, <24 x i32>* %p, align 4
1254  %s1 = shufflevector <24 x i32> %wide.vec, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
1255  %s2 = shufflevector <24 x i32> %wide.vec, <24 x i32> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
1256  %s3 = shufflevector <24 x i32> %wide.vec, <24 x i32> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
1257  store <8 x i32> %s1, <8 x i32>* %q1, align 4
1258  store <8 x i32> %s2, <8 x i32>* %q2, align 4
1259  store <8 x i32> %s3, <8 x i32>* %q3, align 4
1260  ret void
1261}
1262
1263define void @interleave_24i32_in(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2, <8 x i32>* %q3) nounwind {
1264; SSE2-LABEL: interleave_24i32_in:
1265; SSE2:       # BB#0:
1266; SSE2-NEXT:    movdqu (%rsi), %xmm5
1267; SSE2-NEXT:    movdqu 16(%rsi), %xmm2
1268; SSE2-NEXT:    movdqu (%rdx), %xmm6
1269; SSE2-NEXT:    movdqu 16(%rdx), %xmm1
1270; SSE2-NEXT:    movdqu (%rcx), %xmm7
1271; SSE2-NEXT:    movdqu 16(%rcx), %xmm4
1272; SSE2-NEXT:    movdqa %xmm5, %xmm0
1273; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
1274; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,2,2]
1275; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,1,0,1]
1276; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm0[3,0]
1277; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[0,2]
1278; SSE2-NEXT:    movdqa %xmm7, %xmm3
1279; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,0],xmm6[1,0]
1280; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm6[2,2]
1281; SSE2-NEXT:    punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm5[2],xmm7[3],xmm5[3]
1282; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm3[3,0]
1283; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm5[0,2]
1284; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,3,2,2]
1285; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]
1286; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[2,0],xmm5[3,0]
1287; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm6[0,2]
1288; SSE2-NEXT:    movdqa %xmm2, %xmm6
1289; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
1290; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[0,1,2,2]
1291; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,1,0,1]
1292; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[2,0],xmm6[3,0]
1293; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,1],xmm7[0,2]
1294; SSE2-NEXT:    movdqa %xmm4, %xmm7
1295; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[1,0],xmm1[1,0]
1296; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[2,0],xmm1[2,2]
1297; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
1298; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm7[3,0]
1299; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,1],xmm2[0,2]
1300; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,3,2,2]
1301; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
1302; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,0]
1303; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2]
1304; SSE2-NEXT:    movups %xmm2, 80(%rdi)
1305; SSE2-NEXT:    movups %xmm7, 64(%rdi)
1306; SSE2-NEXT:    movups %xmm6, 48(%rdi)
1307; SSE2-NEXT:    movups %xmm5, 32(%rdi)
1308; SSE2-NEXT:    movups %xmm3, 16(%rdi)
1309; SSE2-NEXT:    movups %xmm0, (%rdi)
1310; SSE2-NEXT:    retq
1311;
1312; SSE42-LABEL: interleave_24i32_in:
1313; SSE42:       # BB#0:
1314; SSE42-NEXT:    movdqu (%rsi), %xmm5
1315; SSE42-NEXT:    movdqu 16(%rsi), %xmm2
1316; SSE42-NEXT:    movdqu (%rdx), %xmm6
1317; SSE42-NEXT:    movdqu 16(%rdx), %xmm1
1318; SSE42-NEXT:    movdqu (%rcx), %xmm7
1319; SSE42-NEXT:    movdqu 16(%rcx), %xmm4
1320; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,1,1]
1321; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,1,0,1]
1322; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm0[2,3],xmm3[4,5,6,7]
1323; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,1,0,1]
1324; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5],xmm3[6,7]
1325; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,2,2]
1326; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm7[2,3],xmm3[4,5,6,7]
1327; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm5[4,5],xmm3[6,7]
1328; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,0,1]
1329; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[2,3,2,3]
1330; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm5[2,3],xmm7[4,5,6,7]
1331; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[2,2,3,3]
1332; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm7[0,1,2,3],xmm5[4,5],xmm7[6,7]
1333; SSE42-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,1,1]
1334; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[0,1,0,1]
1335; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm6[2,3],xmm7[4,5,6,7]
1336; SSE42-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,1,0,1]
1337; SSE42-NEXT:    pblendw {{.*#+}} xmm6 = xmm7[0,1,2,3],xmm6[4,5],xmm7[6,7]
1338; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,2,2]
1339; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm4[2,3],xmm7[4,5,6,7]
1340; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1,2,3],xmm2[4,5],xmm7[6,7]
1341; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
1342; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
1343; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3],xmm4[4,5,6,7]
1344; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
1345; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm4[0,1,2,3],xmm1[4,5],xmm4[6,7]
1346; SSE42-NEXT:    movdqu %xmm1, 80(%rdi)
1347; SSE42-NEXT:    movdqu %xmm7, 64(%rdi)
1348; SSE42-NEXT:    movdqu %xmm6, 48(%rdi)
1349; SSE42-NEXT:    movdqu %xmm5, 32(%rdi)
1350; SSE42-NEXT:    movdqu %xmm3, 16(%rdi)
1351; SSE42-NEXT:    movdqu %xmm0, (%rdi)
1352; SSE42-NEXT:    retq
1353;
1354; AVX1-LABEL: interleave_24i32_in:
1355; AVX1:       # BB#0:
1356; AVX1-NEXT:    vmovups (%rsi), %ymm0
1357; AVX1-NEXT:    vmovups (%rdx), %ymm1
1358; AVX1-NEXT:    vmovupd (%rcx), %ymm2
1359; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm0[2,0],xmm1[2,0]
1360; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[1,1],xmm3[0,2]
1361; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[0,0],xmm0[0,0]
1362; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[2,0],xmm0[2,1]
1363; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
1364; AVX1-NEXT:    vmovddup {{.*#+}} xmm4 = xmm2[0,0]
1365; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm4, %ymm4
1366; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
1367; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
1368; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
1369; AVX1-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[3,0],xmm4[3,0]
1370; AVX1-NEXT:    vshufps {{.*#+}} xmm6 = xmm4[2,1],xmm6[0,2]
1371; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[1,0],xmm5[1,0]
1372; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[2,0],xmm5[2,2]
1373; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm4
1374; AVX1-NEXT:    vpermilpd {{.*#+}} ymm5 = ymm0[1,1,3,3]
1375; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm5[2,3,2,3]
1376; AVX1-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]
1377; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,2]
1378; AVX1-NEXT:    vpermilpd {{.*#+}} ymm2 = ymm2[1,1,2,2]
1379; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
1380; AVX1-NEXT:    vpermilps {{.*#+}} ymm1 = ymm1[0,0,3,3,4,4,7,7]
1381; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1382; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
1383; AVX1-NEXT:    vmovups %ymm4, 64(%rdi)
1384; AVX1-NEXT:    vmovups %ymm3, (%rdi)
1385; AVX1-NEXT:    vzeroupper
1386; AVX1-NEXT:    retq
1387;
1388; AVX2-LABEL: interleave_24i32_in:
1389; AVX2:       # BB#0:
1390; AVX2-NEXT:    vmovdqu (%rsi), %ymm0
1391; AVX2-NEXT:    vmovdqu (%rdx), %ymm1
1392; AVX2-NEXT:    vmovdqu (%rcx), %ymm2
1393; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,0,2,2]
1394; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm3[0,1,0,1]
1395; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[0,0,2,1]
1396; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7]
1397; AVX2-NEXT:    vpbroadcastq %xmm2, %ymm4
1398; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
1399; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,1,3,3]
1400; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[1,2,3,3,5,6,7,7]
1401; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
1402; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6],ymm4[7]
1403; AVX2-NEXT:    vpbroadcastq 24(%rsi), %ymm5
1404; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]
1405; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,1,2,2]
1406; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,1,2,2]
1407; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
1408; AVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,3,3,4,4,7,7]
1409; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1410; AVX2-NEXT:    vmovdqu %ymm0, 32(%rdi)
1411; AVX2-NEXT:    vmovdqu %ymm4, 64(%rdi)
1412; AVX2-NEXT:    vmovdqu %ymm3, (%rdi)
1413; AVX2-NEXT:    vzeroupper
1414; AVX2-NEXT:    retq
1415  %s1 = load <8 x i32>, <8 x i32>* %q1, align 4
1416  %s2 = load <8 x i32>, <8 x i32>* %q2, align 4
1417  %s3 = load <8 x i32>, <8 x i32>* %q3, align 4
1418  %t1 = shufflevector <8 x i32> %s1, <8 x i32> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
1419  %t2 = shufflevector <8 x i32> %s3, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1420  %interleaved = shufflevector <16 x i32> %t1, <16 x i32> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
1421  store <24 x i32> %interleaved, <24 x i32>* %p, align 4
1422  ret void
1423}
1424
1425define <2 x double> @wrongorder(<4 x double> %A, <8 x double>* %P) #0 {
1426; SSE2-LABEL: wrongorder:
1427; SSE2:       # BB#0:
1428; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0,0]
1429; SSE2-NEXT:    movaps %xmm0, 48(%rdi)
1430; SSE2-NEXT:    movaps %xmm0, 32(%rdi)
1431; SSE2-NEXT:    movaps %xmm0, 16(%rdi)
1432; SSE2-NEXT:    movaps %xmm0, (%rdi)
1433; SSE2-NEXT:    retq
1434;
1435; SSE42-LABEL: wrongorder:
1436; SSE42:       # BB#0:
1437; SSE42-NEXT:    movddup {{.*#+}} xmm0 = xmm0[0,0]
1438; SSE42-NEXT:    movapd %xmm0, 48(%rdi)
1439; SSE42-NEXT:    movapd %xmm0, 32(%rdi)
1440; SSE42-NEXT:    movapd %xmm0, 16(%rdi)
1441; SSE42-NEXT:    movapd %xmm0, (%rdi)
1442; SSE42-NEXT:    retq
1443;
1444; AVX1-LABEL: wrongorder:
1445; AVX1:       # BB#0:
1446; AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
1447; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
1448; AVX1-NEXT:    vmovapd %ymm1, 32(%rdi)
1449; AVX1-NEXT:    vmovapd %ymm1, (%rdi)
1450; AVX1-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
1451; AVX1-NEXT:    vzeroupper
1452; AVX1-NEXT:    retq
1453;
1454; AVX2-LABEL: wrongorder:
1455; AVX2:       # BB#0:
1456; AVX2-NEXT:    vbroadcastsd %xmm0, %ymm1
1457; AVX2-NEXT:    vmovapd %ymm1, 32(%rdi)
1458; AVX2-NEXT:    vmovapd %ymm1, (%rdi)
1459; AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
1460; AVX2-NEXT:    vzeroupper
1461; AVX2-NEXT:    retq
1462  %shuffle = shufflevector <4 x double> %A, <4 x double> %A, <8 x i32> zeroinitializer
1463  store <8 x double> %shuffle, <8 x double>* %P, align 64
1464  %m2 = load <8 x double>, <8 x double>* %P, align 64
1465  store <8 x double> %m2, <8 x double>* %P, align 64
1466  %m3 = load <8 x double>, <8 x double>* %P, align 64
1467  %m4 = shufflevector <8 x double> %m3, <8 x double> undef, <2 x i32> <i32 2, i32 0>
1468  ret <2 x double> %m4
1469}
1470