1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3           | FileCheck %s --check-prefixes=SSSE3-SLOW
3; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops | FileCheck %s --check-prefixes=SSSE3-FAST
4; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx             | FileCheck %s --check-prefixes=AVX-SLOW,AVX1-SLOW
5; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops   | FileCheck %s --check-prefixes=AVX-FAST,AVX1-FAST
6; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2            | FileCheck %s --check-prefixes=AVX-SLOW,AVX2-SLOW
7; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2,fast-hops  | FileCheck %s --check-prefixes=AVX-FAST,AVX2-FAST
8
9; Vectorized Pairwise Sum Reductions
10; e.g.
11; inline STYPE sum(VTYPE x) {
12;   return (x[0] + x[1]) + (x[2] + x[3]);
13; }
14;
15; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) {
16;   return (VTYPE) { sum( A0 ), sum( A1 ), sum( A2 ), sum( A3 ) };
17; }
18
19define <4 x float> @pair_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {
20; SSSE3-SLOW-LABEL: pair_sum_v4f32_v4f32:
21; SSSE3-SLOW:       # %bb.0:
22; SSSE3-SLOW-NEXT:    haddps %xmm1, %xmm0
23; SSSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
24; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm0[1,3]
25; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3,1,3]
26; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm0
27; SSSE3-SLOW-NEXT:    haddps %xmm2, %xmm2
28; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
29; SSSE3-SLOW-NEXT:    movddup {{.*#+}} xmm2 = xmm2[0,0]
30; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm2
31; SSSE3-SLOW-NEXT:    haddps %xmm3, %xmm3
32; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
33; SSSE3-SLOW-NEXT:    addps %xmm3, %xmm1
34; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3]
35; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,0]
36; SSSE3-SLOW-NEXT:    retq
37;
38; SSSE3-FAST-LABEL: pair_sum_v4f32_v4f32:
39; SSSE3-FAST:       # %bb.0:
40; SSSE3-FAST-NEXT:    haddps %xmm1, %xmm0
41; SSSE3-FAST-NEXT:    haddps %xmm3, %xmm2
42; SSSE3-FAST-NEXT:    haddps %xmm2, %xmm0
43; SSSE3-FAST-NEXT:    retq
44;
45; AVX1-SLOW-LABEL: pair_sum_v4f32_v4f32:
46; AVX1-SLOW:       # %bb.0:
47; AVX1-SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
48; AVX1-SLOW-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
49; AVX1-SLOW-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[0,2],xmm1[0,1]
50; AVX1-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,1]
51; AVX1-SLOW-NEXT:    vhaddps %xmm3, %xmm3, %xmm1
52; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[1]
53; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0]
54; AVX1-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
55; AVX1-SLOW-NEXT:    retq
56;
57; AVX-FAST-LABEL: pair_sum_v4f32_v4f32:
58; AVX-FAST:       # %bb.0:
59; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
60; AVX-FAST-NEXT:    vhaddps %xmm3, %xmm2, %xmm1
61; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
62; AVX-FAST-NEXT:    retq
63;
64; AVX2-SLOW-LABEL: pair_sum_v4f32_v4f32:
65; AVX2-SLOW:       # %bb.0:
66; AVX2-SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
67; AVX2-SLOW-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
68; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[0,2],xmm1[0,3]
69; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,1]
70; AVX2-SLOW-NEXT:    vhaddps %xmm3, %xmm3, %xmm1
71; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[1]
72; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0]
73; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
74; AVX2-SLOW-NEXT:    retq
75  %5 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2>
76  %6 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3>
77  %7 = fadd <2 x float> %5, %6
78  %8 = shufflevector <2 x float> %7, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
79  %9 = fadd <2 x float> %7, %8
80  %10 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 0, i32 2>
81  %11 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 1, i32 3>
82  %12 = fadd <2 x float> %10, %11
83  %13 = shufflevector <2 x float> %12, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
84  %14 = fadd <2 x float> %12, %13
85  %15 = shufflevector <2 x float> %9, <2 x float> %14, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
86  %16 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 0, i32 2>
87  %17 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 1, i32 3>
88  %18 = fadd <2 x float> %16, %17
89  %19 = shufflevector <2 x float> %18, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
90  %20 = fadd <2 x float> %18, %19
91  %21 = shufflevector <2 x float> %20, <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
92  %22 = shufflevector <4 x float> %15, <4 x float> %21, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
93  %23 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 0, i32 2>
94  %24 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 1, i32 3>
95  %25 = fadd <2 x float> %23, %24
96  %26 = shufflevector <2 x float> %25, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
97  %27 = fadd <2 x float> %25, %26
98  %28 = shufflevector <2 x float> %27, <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
99  %29 = shufflevector <4 x float> %22, <4 x float> %28, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
100  ret <4 x float> %29
101}
102
103define <4 x i32> @pair_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {
104; SSSE3-SLOW-LABEL: pair_sum_v4i32_v4i32:
105; SSSE3-SLOW:       # %bb.0:
106; SSSE3-SLOW-NEXT:    phaddd %xmm1, %xmm0
107; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,1,3]
108; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,1,3]
109; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
110; SSSE3-SLOW-NEXT:    phaddd %xmm2, %xmm2
111; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
112; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
113; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm2
114; SSSE3-SLOW-NEXT:    phaddd %xmm3, %xmm3
115; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
116; SSSE3-SLOW-NEXT:    paddd %xmm3, %xmm1
117; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3]
118; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,0]
119; SSSE3-SLOW-NEXT:    retq
120;
121; SSSE3-FAST-LABEL: pair_sum_v4i32_v4i32:
122; SSSE3-FAST:       # %bb.0:
123; SSSE3-FAST-NEXT:    phaddd %xmm3, %xmm2
124; SSSE3-FAST-NEXT:    phaddd %xmm1, %xmm0
125; SSSE3-FAST-NEXT:    phaddd %xmm2, %xmm0
126; SSSE3-FAST-NEXT:    retq
127;
128; AVX1-SLOW-LABEL: pair_sum_v4i32_v4i32:
129; AVX1-SLOW:       # %bb.0:
130; AVX1-SLOW-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
131; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,2,1,3]
132; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,1,3]
133; AVX1-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
134; AVX1-SLOW-NEXT:    vphaddd %xmm2, %xmm2, %xmm1
135; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
136; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
137; AVX1-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
138; AVX1-SLOW-NEXT:    vphaddd %xmm3, %xmm3, %xmm1
139; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
140; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
141; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
142; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
143; AVX1-SLOW-NEXT:    retq
144;
145; AVX-FAST-LABEL: pair_sum_v4i32_v4i32:
146; AVX-FAST:       # %bb.0:
147; AVX-FAST-NEXT:    vphaddd %xmm3, %xmm2, %xmm2
148; AVX-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
149; AVX-FAST-NEXT:    vphaddd %xmm2, %xmm0, %xmm0
150; AVX-FAST-NEXT:    retq
151;
152; AVX2-SLOW-LABEL: pair_sum_v4i32_v4i32:
153; AVX2-SLOW:       # %bb.0:
154; AVX2-SLOW-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
155; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
156; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
157; AVX2-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
158; AVX2-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
159; AVX2-SLOW-NEXT:    vphaddd %xmm2, %xmm2, %xmm1
160; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
161; AVX2-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
162; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3]
163; AVX2-SLOW-NEXT:    vphaddd %xmm3, %xmm3, %xmm1
164; AVX2-SLOW-NEXT:    vpbroadcastq %xmm1, %xmm2
165; AVX2-SLOW-NEXT:    vpbroadcastd %xmm1, %xmm1
166; AVX2-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
167; AVX2-SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
168; AVX2-SLOW-NEXT:    retq
169  %5 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
170  %6 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
171  %7 = add <2 x i32> %5, %6
172  %8 = shufflevector <2 x i32> %7, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
173  %9 = add <2 x i32> %7, %8
174  %10 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
175  %11 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
176  %12 = add <2 x i32> %10, %11
177  %13 = shufflevector <2 x i32> %12, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
178  %14 = add <2 x i32> %12, %13
179  %15 = shufflevector <2 x i32> %9, <2 x i32> %14, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
180  %16 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
181  %17 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
182  %18 = add <2 x i32> %16, %17
183  %19 = shufflevector <2 x i32> %18, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
184  %20 = add <2 x i32> %18, %19
185  %21 = shufflevector <2 x i32> %20, <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
186  %22 = shufflevector <4 x i32> %15, <4 x i32> %21, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
187  %23 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
188  %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
189  %25 = add <2 x i32> %23, %24
190  %26 = shufflevector <2 x i32> %25, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
191  %27 = add <2 x i32> %25, %26
192  %28 = shufflevector <2 x i32> %27, <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
193  %29 = shufflevector <4 x i32> %22, <4 x i32> %28, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
194  ret <4 x i32> %29
195}
196
197define <8 x float> @pair_sum_v8f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3, <4 x float> %4, <4 x float> %5, <4 x float> %6, <4 x float> %7) {
198; SSSE3-SLOW-LABEL: pair_sum_v8f32_v4f32:
199; SSSE3-SLOW:       # %bb.0:
200; SSSE3-SLOW-NEXT:    haddps %xmm1, %xmm0
201; SSSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
202; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm0[1,3]
203; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3,1,3]
204; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm0
205; SSSE3-SLOW-NEXT:    movaps %xmm2, %xmm1
206; SSSE3-SLOW-NEXT:    haddps %xmm3, %xmm1
207; SSSE3-SLOW-NEXT:    haddps %xmm2, %xmm3
208; SSSE3-SLOW-NEXT:    haddps %xmm4, %xmm5
209; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[2,0]
210; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1],xmm5[3,1]
211; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm3
212; SSSE3-SLOW-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]
213; SSSE3-SLOW-NEXT:    haddps %xmm6, %xmm6
214; SSSE3-SLOW-NEXT:    haddps %xmm7, %xmm7
215; SSSE3-SLOW-NEXT:    haddps %xmm7, %xmm6
216; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,3],xmm6[0,2]
217; SSSE3-SLOW-NEXT:    movaps %xmm3, %xmm1
218; SSSE3-SLOW-NEXT:    retq
219;
220; SSSE3-FAST-LABEL: pair_sum_v8f32_v4f32:
221; SSSE3-FAST:       # %bb.0:
222; SSSE3-FAST-NEXT:    haddps %xmm1, %xmm0
223; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm0
224; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm1
225; SSSE3-FAST-NEXT:    haddps %xmm3, %xmm1
226; SSSE3-FAST-NEXT:    haddps %xmm2, %xmm3
227; SSSE3-FAST-NEXT:    haddps %xmm4, %xmm5
228; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[2,0]
229; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1],xmm5[3,1]
230; SSSE3-FAST-NEXT:    addps %xmm1, %xmm3
231; SSSE3-FAST-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]
232; SSSE3-FAST-NEXT:    haddps %xmm6, %xmm6
233; SSSE3-FAST-NEXT:    haddps %xmm7, %xmm7
234; SSSE3-FAST-NEXT:    haddps %xmm7, %xmm6
235; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,3],xmm6[0,2]
236; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm1
237; SSSE3-FAST-NEXT:    retq
238;
239; AVX1-SLOW-LABEL: pair_sum_v8f32_v4f32:
240; AVX1-SLOW:       # %bb.0:
241; AVX1-SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
242; AVX1-SLOW-NEXT:    vpermilps {{.*#+}} xmm8 = xmm0[0,2,1,3]
243; AVX1-SLOW-NEXT:    vxorps %xmm1, %xmm1, %xmm1
244; AVX1-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[0,1]
245; AVX1-SLOW-NEXT:    vaddps %xmm0, %xmm8, %xmm0
246; AVX1-SLOW-NEXT:    vhaddps %xmm3, %xmm2, %xmm1
247; AVX1-SLOW-NEXT:    vhaddps %xmm4, %xmm4, %xmm2
248; AVX1-SLOW-NEXT:    vhaddps %xmm5, %xmm5, %xmm3
249; AVX1-SLOW-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[0,2],xmm2[0,1]
250; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],xmm3[0]
251; AVX1-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,3],xmm2[1,1]
252; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[1]
253; AVX1-SLOW-NEXT:    vaddps %xmm1, %xmm4, %xmm1
254; AVX1-SLOW-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
255; AVX1-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
256; AVX1-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
257; AVX1-SLOW-NEXT:    vhaddps %xmm6, %xmm6, %xmm2
258; AVX1-SLOW-NEXT:    vhaddps %xmm7, %xmm7, %xmm3
259; AVX1-SLOW-NEXT:    vhaddps %xmm3, %xmm2, %xmm2
260; AVX1-SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3]
261; AVX1-SLOW-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
262; AVX1-SLOW-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
263; AVX1-SLOW-NEXT:    retq
264;
265; AVX1-FAST-LABEL: pair_sum_v8f32_v4f32:
266; AVX1-FAST:       # %bb.0:
267; AVX1-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
268; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
269; AVX1-FAST-NEXT:    vhaddps %xmm3, %xmm2, %xmm1
270; AVX1-FAST-NEXT:    vhaddps %xmm2, %xmm2, %xmm2
271; AVX1-FAST-NEXT:    vhaddps %xmm3, %xmm3, %xmm3
272; AVX1-FAST-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[1],xmm3[1],zero,zero
273; AVX1-FAST-NEXT:    vhaddps %xmm4, %xmm4, %xmm3
274; AVX1-FAST-NEXT:    vhaddps %xmm5, %xmm5, %xmm4
275; AVX1-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,1]
276; AVX1-FAST-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]
277; AVX1-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[1,3]
278; AVX1-FAST-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[1]
279; AVX1-FAST-NEXT:    vaddps %xmm2, %xmm1, %xmm1
280; AVX1-FAST-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
281; AVX1-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
282; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
283; AVX1-FAST-NEXT:    vhaddps %xmm7, %xmm6, %xmm2
284; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm2, %xmm2
285; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
286; AVX1-FAST-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
287; AVX1-FAST-NEXT:    retq
288;
289; AVX2-SLOW-LABEL: pair_sum_v8f32_v4f32:
290; AVX2-SLOW:       # %bb.0:
291; AVX2-SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
292; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm8 = xmm0[0,2,1,3]
293; AVX2-SLOW-NEXT:    vxorps %xmm1, %xmm1, %xmm1
294; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[0,1]
295; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm8, %xmm0
296; AVX2-SLOW-NEXT:    vhaddps %xmm4, %xmm4, %xmm1
297; AVX2-SLOW-NEXT:    vhaddps %xmm5, %xmm5, %xmm4
298; AVX2-SLOW-NEXT:    vhaddps %xmm3, %xmm2, %xmm2
299; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3]
300; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
301; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
302; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1]
303; AVX2-SLOW-NEXT:    vaddps %xmm1, %xmm3, %xmm1
304; AVX2-SLOW-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
305; AVX2-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
306; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
307; AVX2-SLOW-NEXT:    vhaddps %xmm6, %xmm6, %xmm2
308; AVX2-SLOW-NEXT:    vhaddps %xmm7, %xmm7, %xmm3
309; AVX2-SLOW-NEXT:    vhaddps %xmm3, %xmm2, %xmm2
310; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3]
311; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
312; AVX2-SLOW-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
313; AVX2-SLOW-NEXT:    retq
314;
315; AVX2-FAST-LABEL: pair_sum_v8f32_v4f32:
316; AVX2-FAST:       # %bb.0:
317; AVX2-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
318; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
319; AVX2-FAST-NEXT:    vhaddps %xmm4, %xmm4, %xmm1
320; AVX2-FAST-NEXT:    vhaddps %xmm5, %xmm5, %xmm4
321; AVX2-FAST-NEXT:    vhaddps %xmm3, %xmm2, %xmm2
322; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3]
323; AVX2-FAST-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
324; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
325; AVX2-FAST-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1]
326; AVX2-FAST-NEXT:    vaddps %xmm1, %xmm3, %xmm1
327; AVX2-FAST-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
328; AVX2-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
329; AVX2-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
330; AVX2-FAST-NEXT:    vhaddps %xmm7, %xmm6, %xmm2
331; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm2, %xmm2
332; AVX2-FAST-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
333; AVX2-FAST-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
334; AVX2-FAST-NEXT:    retq
335  %9 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2>
336  %10 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3>
337  %11 = fadd <2 x float> %9, %10
338  %12 = shufflevector <2 x float> %11, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
339  %13 = fadd <2 x float> %11, %12
340  %14 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 0, i32 2>
341  %15 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 1, i32 3>
342  %16 = fadd <2 x float> %14, %15
343  %17 = shufflevector <2 x float> %16, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
344  %18 = fadd <2 x float> %16, %17
345  %19 = shufflevector <2 x float> %13, <2 x float> %18, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
346  %20 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 0, i32 2>
347  %21 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 1, i32 3>
348  %22 = fadd <2 x float> %20, %21
349  %23 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 0, i32 2>
350  %24 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 1, i32 3>
351  %25 = fadd <2 x float> %23, %24
352  %26 = shufflevector <4 x float> %4, <4 x float> poison, <2 x i32> <i32 0, i32 2>
353  %27 = shufflevector <4 x float> %4, <4 x float> poison, <2 x i32> <i32 1, i32 3>
354  %28 = fadd <2 x float> %26, %27
355  %29 = shufflevector <2 x float> %28, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
356  %30 = shufflevector <4 x float> %5, <4 x float> poison, <2 x i32> <i32 0, i32 2>
357  %31 = shufflevector <4 x float> %5, <4 x float> poison, <2 x i32> <i32 1, i32 3>
358  %32 = fadd <2 x float> %30, %31
359  %33 = shufflevector <2 x float> %32, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
360  %34 = shufflevector <2 x float> %22, <2 x float> %25, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
361  %35 = shufflevector <4 x float> %34, <4 x float> %29, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
362  %36 = shufflevector <4 x float> %35, <4 x float> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
363  %37 = shufflevector <2 x float> %22, <2 x float> %25, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
364  %38 = shufflevector <4 x float> %37, <4 x float> %29, <4 x i32> <i32 0, i32 1, i32 5, i32 undef>
365  %39 = shufflevector <4 x float> %38, <4 x float> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 5>
366  %40 = fadd <4 x float> %36, %39
367  %41 = shufflevector <4 x float> %40, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
368  %42 = shufflevector <8 x float> %19, <8 x float> %41, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef>
369  %43 = shufflevector <4 x float> %6, <4 x float> poison, <2 x i32> <i32 0, i32 2>
370  %44 = shufflevector <4 x float> %6, <4 x float> poison, <2 x i32> <i32 1, i32 3>
371  %45 = fadd <2 x float> %43, %44
372  %46 = shufflevector <4 x float> %7, <4 x float> poison, <2 x i32> <i32 0, i32 2>
373  %47 = shufflevector <4 x float> %7, <4 x float> poison, <2 x i32> <i32 1, i32 3>
374  %48 = fadd <2 x float> %46, %47
375  %49 = shufflevector <2 x float> %45, <2 x float> %48, <2 x i32> <i32 0, i32 2>
376  %50 = shufflevector <2 x float> %45, <2 x float> %48, <2 x i32> <i32 1, i32 3>
377  %51 = fadd <2 x float> %49, %50
378  %52 = shufflevector <2 x float> %51, <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
379  %53 = shufflevector <8 x float> %42, <8 x float> %52, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
380  ret <8 x float> %53
381}
382
383define <8 x i32> @pair_sum_v8i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3, <4 x i32> %4, <4 x i32> %5, <4 x i32> %6, <4 x i32> %7) {
384; SSSE3-SLOW-LABEL: pair_sum_v8i32_v4i32:
385; SSSE3-SLOW:       # %bb.0:
386; SSSE3-SLOW-NEXT:    phaddd %xmm1, %xmm0
387; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,1,3]
388; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,1,3]
389; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
390; SSSE3-SLOW-NEXT:    phaddd %xmm3, %xmm2
391; SSSE3-SLOW-NEXT:    phaddd %xmm4, %xmm5
392; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[2,3,2,3]
393; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,1,0,1]
394; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[1,1]
395; SSSE3-SLOW-NEXT:    movdqa %xmm2, %xmm1
396; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[2,0]
397; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,3],xmm3[2,0]
398; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm2
399; SSSE3-SLOW-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
400; SSSE3-SLOW-NEXT:    phaddd %xmm6, %xmm6
401; SSSE3-SLOW-NEXT:    phaddd %xmm7, %xmm7
402; SSSE3-SLOW-NEXT:    phaddd %xmm7, %xmm6
403; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,3],xmm6[0,2]
404; SSSE3-SLOW-NEXT:    movaps %xmm2, %xmm1
405; SSSE3-SLOW-NEXT:    retq
406;
407; SSSE3-FAST-LABEL: pair_sum_v8i32_v4i32:
408; SSSE3-FAST:       # %bb.0:
409; SSSE3-FAST-NEXT:    phaddd %xmm1, %xmm0
410; SSSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
411; SSSE3-FAST-NEXT:    phaddd %xmm3, %xmm2
412; SSSE3-FAST-NEXT:    movdqa %xmm5, %xmm1
413; SSSE3-FAST-NEXT:    phaddd %xmm4, %xmm5
414; SSSE3-FAST-NEXT:    phaddd %xmm4, %xmm4
415; SSSE3-FAST-NEXT:    phaddd %xmm1, %xmm1
416; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[1,1]
417; SSSE3-FAST-NEXT:    movdqa %xmm2, %xmm3
418; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm5[2,0]
419; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[2,0]
420; SSSE3-FAST-NEXT:    paddd %xmm3, %xmm2
421; SSSE3-FAST-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
422; SSSE3-FAST-NEXT:    phaddd %xmm6, %xmm6
423; SSSE3-FAST-NEXT:    phaddd %xmm7, %xmm7
424; SSSE3-FAST-NEXT:    phaddd %xmm7, %xmm6
425; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,3],xmm6[0,2]
426; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm1
427; SSSE3-FAST-NEXT:    retq
428;
429; AVX1-SLOW-LABEL: pair_sum_v8i32_v4i32:
430; AVX1-SLOW:       # %bb.0:
431; AVX1-SLOW-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
432; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,2,1,3]
433; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,1,1]
434; AVX1-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
435; AVX1-SLOW-NEXT:    vphaddd %xmm3, %xmm2, %xmm1
436; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,2,1,3]
437; AVX1-SLOW-NEXT:    vphaddd %xmm4, %xmm4, %xmm3
438; AVX1-SLOW-NEXT:    vphaddd %xmm5, %xmm5, %xmm4
439; AVX1-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
440; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[0,0,0,0]
441; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5],xmm5[6,7]
442; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,3,1,1]
443; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[1],zero
444; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1]
445; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
446; AVX1-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
447; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
448; AVX1-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
449; AVX1-SLOW-NEXT:    vphaddd %xmm6, %xmm6, %xmm2
450; AVX1-SLOW-NEXT:    vphaddd %xmm7, %xmm7, %xmm3
451; AVX1-SLOW-NEXT:    vphaddd %xmm3, %xmm2, %xmm2
452; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
453; AVX1-SLOW-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
454; AVX1-SLOW-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
455; AVX1-SLOW-NEXT:    retq
456;
457; AVX1-FAST-LABEL: pair_sum_v8i32_v4i32:
458; AVX1-FAST:       # %bb.0:
459; AVX1-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
460; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm8
461; AVX1-FAST-NEXT:    vphaddd %xmm3, %xmm2, %xmm1
462; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
463; AVX1-FAST-NEXT:    vphaddd %xmm2, %xmm2, %xmm2
464; AVX1-FAST-NEXT:    vphaddd %xmm3, %xmm3, %xmm3
465; AVX1-FAST-NEXT:    vphaddd %xmm4, %xmm4, %xmm4
466; AVX1-FAST-NEXT:    vphaddd %xmm5, %xmm5, %xmm5
467; AVX1-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
468; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm5[0,0,0,0]
469; AVX1-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
470; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
471; AVX1-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]
472; AVX1-FAST-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm4[1],zero
473; AVX1-FAST-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[3]
474; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
475; AVX1-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm8[0],xmm0[0]
476; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
477; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
478; AVX1-FAST-NEXT:    vphaddd %xmm7, %xmm6, %xmm2
479; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm2, %xmm2
480; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
481; AVX1-FAST-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2]
482; AVX1-FAST-NEXT:    retq
483;
484; AVX2-SLOW-LABEL: pair_sum_v8i32_v4i32:
485; AVX2-SLOW:       # %bb.0:
486; AVX2-SLOW-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
487; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,2,1,3]
488; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,1,1]
489; AVX2-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
490; AVX2-SLOW-NEXT:    vphaddd %xmm4, %xmm4, %xmm1
491; AVX2-SLOW-NEXT:    vphaddd %xmm5, %xmm5, %xmm4
492; AVX2-SLOW-NEXT:    vphaddd %xmm3, %xmm2, %xmm2
493; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3]
494; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
495; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
496; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1]
497; AVX2-SLOW-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
498; AVX2-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
499; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
500; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
501; AVX2-SLOW-NEXT:    vphaddd %xmm6, %xmm6, %xmm1
502; AVX2-SLOW-NEXT:    vphaddd %xmm7, %xmm7, %xmm2
503; AVX2-SLOW-NEXT:    vphaddd %xmm2, %xmm1, %xmm1
504; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
505; AVX2-SLOW-NEXT:    vpbroadcastq %xmm1, %ymm1
506; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
507; AVX2-SLOW-NEXT:    retq
508;
509; AVX2-FAST-LABEL: pair_sum_v8i32_v4i32:
510; AVX2-FAST:       # %bb.0:
511; AVX2-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
512; AVX2-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm1
513; AVX2-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
514; AVX2-FAST-NEXT:    vphaddd %xmm4, %xmm4, %xmm4
515; AVX2-FAST-NEXT:    vphaddd %xmm5, %xmm5, %xmm5
516; AVX2-FAST-NEXT:    vphaddd %xmm3, %xmm2, %xmm2
517; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm4[0,3]
518; AVX2-FAST-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm5[0]
519; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[1,3],xmm4[1,3]
520; AVX2-FAST-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0,1,2],xmm5[3]
521; AVX2-FAST-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
522; AVX2-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
523; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
524; AVX2-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
525; AVX2-FAST-NEXT:    vphaddd %xmm7, %xmm6, %xmm2
526; AVX2-FAST-NEXT:    vphaddd %xmm1, %xmm2, %xmm1
527; AVX2-FAST-NEXT:    vpbroadcastq %xmm1, %ymm1
528; AVX2-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
529; AVX2-FAST-NEXT:    retq
530  %9 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
531  %10 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
532  %11 = add <2 x i32> %9, %10
533  %12 = shufflevector <2 x i32> %11, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
534  %13 = add <2 x i32> %11, %12
535  %14 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
536  %15 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
537  %16 = add <2 x i32> %14, %15
538  %17 = shufflevector <2 x i32> %16, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
539  %18 = add <2 x i32> %16, %17
540  %19 = shufflevector <2 x i32> %13, <2 x i32> %18, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
541  %20 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
542  %21 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
543  %22 = add <2 x i32> %20, %21
544  %23 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
545  %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
546  %25 = add <2 x i32> %23, %24
547  %26 = shufflevector <4 x i32> %4, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
548  %27 = shufflevector <4 x i32> %4, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
549  %28 = add <2 x i32> %26, %27
550  %29 = shufflevector <2 x i32> %28, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
551  %30 = shufflevector <4 x i32> %5, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
552  %31 = shufflevector <4 x i32> %5, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
553  %32 = add <2 x i32> %30, %31
554  %33 = shufflevector <2 x i32> %32, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
555  %34 = shufflevector <2 x i32> %22, <2 x i32> %25, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
556  %35 = shufflevector <4 x i32> %34, <4 x i32> %29, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
557  %36 = shufflevector <4 x i32> %35, <4 x i32> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
558  %37 = shufflevector <2 x i32> %22, <2 x i32> %25, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
559  %38 = shufflevector <4 x i32> %37, <4 x i32> %29, <4 x i32> <i32 0, i32 1, i32 5, i32 undef>
560  %39 = shufflevector <4 x i32> %38, <4 x i32> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 5>
561  %40 = add <4 x i32> %36, %39
562  %41 = shufflevector <4 x i32> %40, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
563  %42 = shufflevector <8 x i32> %19, <8 x i32> %41, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef>
564  %43 = shufflevector <4 x i32> %6, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
565  %44 = shufflevector <4 x i32> %6, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
566  %45 = add <2 x i32> %43, %44
567  %46 = shufflevector <4 x i32> %7, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
568  %47 = shufflevector <4 x i32> %7, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
569  %48 = add <2 x i32> %46, %47
570  %49 = shufflevector <2 x i32> %45, <2 x i32> %48, <2 x i32> <i32 0, i32 2>
571  %50 = shufflevector <2 x i32> %45, <2 x i32> %48, <2 x i32> <i32 1, i32 3>
572  %51 = add <2 x i32> %49, %50
573  %52 = shufflevector <2 x i32> %51, <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
574  %53 = shufflevector <8 x i32> %42, <8 x i32> %52, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
575  ret <8 x i32> %53
576}
577
578; Vectorized Sequential Sum Reductions
579; e.g.
580; inline STYPE sum(VTYPE x) {
581;   return ((x[0] + x[1]) + x[2]) + x[3];
582; }
583;
584; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) {
585;   return (VTYPE) { sum( A0 ), sum( A1 ), sum( A2 ), sum( A3 ) };
586; }
587
588define <4 x float> @sequential_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {
589; SSSE3-SLOW-LABEL: sequential_sum_v4f32_v4f32:
590; SSSE3-SLOW:       # %bb.0:
591; SSSE3-SLOW-NEXT:    movaps %xmm0, %xmm4
592; SSSE3-SLOW-NEXT:    haddps %xmm1, %xmm4
593; SSSE3-SLOW-NEXT:    movaps %xmm0, %xmm5
594; SSSE3-SLOW-NEXT:    unpckhps {{.*#+}} xmm5 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]
595; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3]
596; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm0 = xmm2[1,1,3,3]
597; SSSE3-SLOW-NEXT:    addps %xmm2, %xmm0
598; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,2],xmm0[0,1]
599; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm2[2,3]
600; SSSE3-SLOW-NEXT:    addps %xmm4, %xmm5
601; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,3]
602; SSSE3-SLOW-NEXT:    addps %xmm5, %xmm1
603; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm0 = xmm3[1,1,3,3]
604; SSSE3-SLOW-NEXT:    addps %xmm3, %xmm0
605; SSSE3-SLOW-NEXT:    movaps %xmm3, %xmm2
606; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1]
607; SSSE3-SLOW-NEXT:    addps %xmm0, %xmm2
608; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
609; SSSE3-SLOW-NEXT:    addps %xmm2, %xmm3
610; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm1[2,3]
611; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,0]
612; SSSE3-SLOW-NEXT:    movaps %xmm1, %xmm0
613; SSSE3-SLOW-NEXT:    retq
614;
615; SSSE3-FAST-LABEL: sequential_sum_v4f32_v4f32:
616; SSSE3-FAST:       # %bb.0:
617; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm4
618; SSSE3-FAST-NEXT:    haddps %xmm1, %xmm4
619; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm5
620; SSSE3-FAST-NEXT:    unpckhps {{.*#+}} xmm5 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]
621; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3]
622; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm2[2,3]
623; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,3]
624; SSSE3-FAST-NEXT:    haddps %xmm2, %xmm2
625; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,2],xmm2[0,1]
626; SSSE3-FAST-NEXT:    addps %xmm4, %xmm5
627; SSSE3-FAST-NEXT:    addps %xmm5, %xmm1
628; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm0
629; SSSE3-FAST-NEXT:    haddps %xmm3, %xmm0
630; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm2
631; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1]
632; SSSE3-FAST-NEXT:    addps %xmm0, %xmm2
633; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
634; SSSE3-FAST-NEXT:    addps %xmm2, %xmm3
635; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm1[2,3]
636; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,0]
637; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm0
638; SSSE3-FAST-NEXT:    retq
639;
640; AVX-SLOW-LABEL: sequential_sum_v4f32_v4f32:
641; AVX-SLOW:       # %bb.0:
642; AVX-SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm4
643; AVX-SLOW-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
644; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
645; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
646; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
647; AVX-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
648; AVX-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm4[0,2],xmm1[0,1]
649; AVX-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[3,3]
650; AVX-SLOW-NEXT:    vblendps {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
651; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3]
652; AVX-SLOW-NEXT:    vaddps %xmm3, %xmm4, %xmm4
653; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]
654; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[2]
655; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm2, %xmm1
656; AVX-SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3]
657; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
658; AVX-SLOW-NEXT:    retq
659;
660; AVX-FAST-LABEL: sequential_sum_v4f32_v4f32:
661; AVX-FAST:       # %bb.0:
662; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm4
663; AVX-FAST-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
664; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
665; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
666; AVX-FAST-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
667; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm4[0,2],xmm1[0,1]
668; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[3,3]
669; AVX-FAST-NEXT:    vblendps {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
670; AVX-FAST-NEXT:    vhaddps %xmm3, %xmm3, %xmm4
671; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]
672; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[2]
673; AVX-FAST-NEXT:    vaddps %xmm1, %xmm2, %xmm1
674; AVX-FAST-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3]
675; AVX-FAST-NEXT:    vaddps %xmm1, %xmm0, %xmm0
676; AVX-FAST-NEXT:    retq
677  %5 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 0, i32 4>
678  %6 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 1, i32 5>
679  %7 = fadd <2 x float> %5, %6
680  %8 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 2, i32 6>
681  %9 = fadd <2 x float> %8, %7
682  %10 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 3, i32 7>
683  %11 = fadd <2 x float> %10, %9
684  %12 = shufflevector <2 x float> %11, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
685  %13 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
686  %14 = fadd <4 x float> %13, %2
687  %15 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
688  %16 = fadd <4 x float> %15, %14
689  %17 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
690  %18 = fadd <4 x float> %17, %16
691  %19 = shufflevector <4 x float> %12, <4 x float> %18, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
692  %20 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
693  %21 = fadd <4 x float> %20, %3
694  %22 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
695  %23 = fadd <4 x float> %22, %21
696  %24 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
697  %25 = fadd <4 x float> %24, %23
698  %26 = shufflevector <4 x float> %19, <4 x float> %25, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
699  ret <4 x float> %26
700}
701
702define <4 x i32> @sequential_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {
703; SSSE3-SLOW-LABEL: sequential_sum_v4i32_v4i32:
704; SSSE3-SLOW:       # %bb.0:
705; SSSE3-SLOW-NEXT:    movdqa %xmm0, %xmm4
706; SSSE3-SLOW-NEXT:    phaddd %xmm1, %xmm4
707; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
708; SSSE3-SLOW-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
709; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
710; SSSE3-SLOW-NEXT:    paddd %xmm4, %xmm1
711; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
712; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
713; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]
714; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,1,0,1]
715; SSSE3-SLOW-NEXT:    paddd %xmm2, %xmm5
716; SSSE3-SLOW-NEXT:    paddd %xmm4, %xmm5
717; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm5
718; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
719; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
720; SSSE3-SLOW-NEXT:    paddd %xmm3, %xmm2
721; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
722; SSSE3-SLOW-NEXT:    paddd %xmm2, %xmm3
723; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm3
724; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm5[2,3]
725; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
726; SSSE3-SLOW-NEXT:    retq
727;
728; SSSE3-FAST-LABEL: sequential_sum_v4i32_v4i32:
729; SSSE3-FAST:       # %bb.0:
730; SSSE3-FAST-NEXT:    movdqa %xmm0, %xmm4
731; SSSE3-FAST-NEXT:    phaddd %xmm1, %xmm4
732; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
733; SSSE3-FAST-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
734; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
735; SSSE3-FAST-NEXT:    paddd %xmm4, %xmm1
736; SSSE3-FAST-NEXT:    paddd %xmm1, %xmm0
737; SSSE3-FAST-NEXT:    movdqa %xmm2, %xmm1
738; SSSE3-FAST-NEXT:    phaddd %xmm2, %xmm1
739; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]
740; SSSE3-FAST-NEXT:    paddd %xmm1, %xmm4
741; SSSE3-FAST-NEXT:    paddd %xmm2, %xmm4
742; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
743; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[3,3,3,3]
744; SSSE3-FAST-NEXT:    phaddd %xmm3, %xmm3
745; SSSE3-FAST-NEXT:    paddd %xmm3, %xmm2
746; SSSE3-FAST-NEXT:    paddd %xmm1, %xmm2
747; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3]
748; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
749; SSSE3-FAST-NEXT:    retq
750;
751; AVX1-SLOW-LABEL: sequential_sum_v4i32_v4i32:
752; AVX1-SLOW:       # %bb.0:
753; AVX1-SLOW-NEXT:    vphaddd %xmm1, %xmm0, %xmm4
754; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
755; AVX1-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
756; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
757; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
758; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
759; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
760; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
761; AVX1-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0]
762; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]
763; AVX1-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
764; AVX1-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
765; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm5[0,1,2,3],xmm2[4,5,6,7]
766; AVX1-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
767; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
768; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[0,0,0,0]
769; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,2,2,2]
770; AVX1-SLOW-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
771; AVX1-SLOW-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
772; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
773; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
774; AVX1-SLOW-NEXT:    retq
775;
776; AVX1-FAST-LABEL: sequential_sum_v4i32_v4i32:
777; AVX1-FAST:       # %bb.0:
778; AVX1-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm4
779; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
780; AVX1-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
781; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
782; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
783; AVX1-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
784; AVX1-FAST-NEXT:    vphaddd %xmm2, %xmm2, %xmm1
785; AVX1-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0]
786; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]
787; AVX1-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
788; AVX1-FAST-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
789; AVX1-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm5[0,1,2,3],xmm2[4,5,6,7]
790; AVX1-FAST-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
791; AVX1-FAST-NEXT:    vphaddd %xmm3, %xmm3, %xmm1
792; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
793; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
794; AVX1-FAST-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
795; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
796; AVX1-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
797; AVX1-FAST-NEXT:    retq
798;
799; AVX2-SLOW-LABEL: sequential_sum_v4i32_v4i32:
800; AVX2-SLOW:       # %bb.0:
801; AVX2-SLOW-NEXT:    vphaddd %xmm1, %xmm0, %xmm4
802; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
803; AVX2-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
804; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
805; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
806; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
807; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
808; AVX2-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
809; AVX2-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0]
810; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]
811; AVX2-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
812; AVX2-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
813; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm5[0,1],xmm2[2,3]
814; AVX2-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
815; AVX2-SLOW-NEXT:    vpbroadcastq %xmm3, %xmm1
816; AVX2-SLOW-NEXT:    vpbroadcastd %xmm3, %xmm2
817; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,2,2,2]
818; AVX2-SLOW-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
819; AVX2-SLOW-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
820; AVX2-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
821; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
822; AVX2-SLOW-NEXT:    retq
823;
824; AVX2-FAST-LABEL: sequential_sum_v4i32_v4i32:
825; AVX2-FAST:       # %bb.0:
826; AVX2-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm4
827; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
828; AVX2-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
829; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
830; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
831; AVX2-FAST-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
832; AVX2-FAST-NEXT:    vphaddd %xmm2, %xmm2, %xmm1
833; AVX2-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0]
834; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]
835; AVX2-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
836; AVX2-FAST-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
837; AVX2-FAST-NEXT:    vpblendd {{.*#+}} xmm1 = xmm5[0,1],xmm2[2,3]
838; AVX2-FAST-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
839; AVX2-FAST-NEXT:    vphaddd %xmm3, %xmm3, %xmm1
840; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
841; AVX2-FAST-NEXT:    vpbroadcastd %xmm1, %xmm1
842; AVX2-FAST-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
843; AVX2-FAST-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
844; AVX2-FAST-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
845; AVX2-FAST-NEXT:    retq
846  %5 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 0, i32 4>
847  %6 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 1, i32 5>
848  %7 = add <2 x i32> %5, %6
849  %8 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 2, i32 6>
850  %9 = add <2 x i32> %8, %7
851  %10 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 3, i32 7>
852  %11 = add <2 x i32> %10, %9
853  %12 = shufflevector <2 x i32> %11, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
854  %13 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
855  %14 = add <4 x i32> %13, %2
856  %15 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
857  %16 = add <4 x i32> %15, %14
858  %17 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
859  %18 = add <4 x i32> %17, %16
860  %19 = shufflevector <4 x i32> %12, <4 x i32> %18, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
861  %20 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
862  %21 = add <4 x i32> %20, %3
863  %22 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
864  %23 = add <4 x i32> %22, %21
865  %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
866  %25 = add <4 x i32> %24, %23
867  %26 = shufflevector <4 x i32> %19, <4 x i32> %25, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
868  ret <4 x i32> %26
869}
870
871; Vectorized Reductions
872; e.g.
873; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) {
874;   return (VTYPE) { reduce( A0 ), reduce( A1 ), reduce( A2 ), reduce( A3 ) };
875; }
876
877define <4 x float> @reduction_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {
878; SSSE3-SLOW-LABEL: reduction_sum_v4f32_v4f32:
879; SSSE3-SLOW:       # %bb.0:
880; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
881; SSSE3-SLOW-NEXT:    addss %xmm0, %xmm4
882; SSSE3-SLOW-NEXT:    movaps %xmm0, %xmm5
883; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]
884; SSSE3-SLOW-NEXT:    addss %xmm4, %xmm5
885; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
886; SSSE3-SLOW-NEXT:    addss %xmm5, %xmm0
887; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
888; SSSE3-SLOW-NEXT:    addss %xmm1, %xmm4
889; SSSE3-SLOW-NEXT:    movaps %xmm1, %xmm5
890; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]
891; SSSE3-SLOW-NEXT:    addss %xmm4, %xmm5
892; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
893; SSSE3-SLOW-NEXT:    addss %xmm5, %xmm1
894; SSSE3-SLOW-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
895; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
896; SSSE3-SLOW-NEXT:    addss %xmm2, %xmm1
897; SSSE3-SLOW-NEXT:    movaps %xmm2, %xmm4
898; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]
899; SSSE3-SLOW-NEXT:    addss %xmm1, %xmm4
900; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
901; SSSE3-SLOW-NEXT:    addss %xmm4, %xmm2
902; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
903; SSSE3-SLOW-NEXT:    addss %xmm3, %xmm1
904; SSSE3-SLOW-NEXT:    movaps %xmm3, %xmm4
905; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
906; SSSE3-SLOW-NEXT:    addss %xmm1, %xmm4
907; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
908; SSSE3-SLOW-NEXT:    addss %xmm4, %xmm3
909; SSSE3-SLOW-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
910; SSSE3-SLOW-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
911; SSSE3-SLOW-NEXT:    retq
912;
913; SSSE3-FAST-LABEL: reduction_sum_v4f32_v4f32:
914; SSSE3-FAST:       # %bb.0:
915; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm4
916; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm4
917; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm5
918; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]
919; SSSE3-FAST-NEXT:    addss %xmm4, %xmm5
920; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
921; SSSE3-FAST-NEXT:    addss %xmm5, %xmm0
922; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm4
923; SSSE3-FAST-NEXT:    haddps %xmm1, %xmm4
924; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm5
925; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]
926; SSSE3-FAST-NEXT:    addss %xmm4, %xmm5
927; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
928; SSSE3-FAST-NEXT:    addss %xmm5, %xmm1
929; SSSE3-FAST-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
930; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm1
931; SSSE3-FAST-NEXT:    haddps %xmm2, %xmm1
932; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm4
933; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]
934; SSSE3-FAST-NEXT:    addss %xmm1, %xmm4
935; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
936; SSSE3-FAST-NEXT:    addss %xmm4, %xmm2
937; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm1
938; SSSE3-FAST-NEXT:    haddps %xmm3, %xmm1
939; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm4
940; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
941; SSSE3-FAST-NEXT:    addss %xmm1, %xmm4
942; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
943; SSSE3-FAST-NEXT:    addss %xmm4, %xmm3
944; SSSE3-FAST-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
945; SSSE3-FAST-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
946; SSSE3-FAST-NEXT:    retq
947;
948; AVX-SLOW-LABEL: reduction_sum_v4f32_v4f32:
949; AVX-SLOW:       # %bb.0:
950; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
951; AVX-SLOW-NEXT:    vaddss %xmm4, %xmm0, %xmm4
952; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm5 = xmm0[1,0]
953; AVX-SLOW-NEXT:    vaddss %xmm5, %xmm4, %xmm4
954; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
955; AVX-SLOW-NEXT:    vaddss %xmm0, %xmm4, %xmm0
956; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
957; AVX-SLOW-NEXT:    vaddss %xmm4, %xmm1, %xmm4
958; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm5 = xmm1[1,0]
959; AVX-SLOW-NEXT:    vaddss %xmm5, %xmm4, %xmm4
960; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
961; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm4, %xmm1
962; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
963; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
964; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm2, %xmm1
965; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
966; AVX-SLOW-NEXT:    vaddss %xmm4, %xmm1, %xmm1
967; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
968; AVX-SLOW-NEXT:    vaddss %xmm2, %xmm1, %xmm1
969; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
970; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
971; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm3, %xmm1
972; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
973; AVX-SLOW-NEXT:    vaddss %xmm2, %xmm1, %xmm1
974; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3]
975; AVX-SLOW-NEXT:    vaddss %xmm2, %xmm1, %xmm1
976; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
977; AVX-SLOW-NEXT:    retq
978;
979; AVX-FAST-LABEL: reduction_sum_v4f32_v4f32:
980; AVX-FAST:       # %bb.0:
981; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm4
982; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm5 = xmm0[1,0]
983; AVX-FAST-NEXT:    vaddss %xmm5, %xmm4, %xmm4
984; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
985; AVX-FAST-NEXT:    vaddss %xmm0, %xmm4, %xmm0
986; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm4
987; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm5 = xmm1[1,0]
988; AVX-FAST-NEXT:    vaddss %xmm5, %xmm4, %xmm4
989; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
990; AVX-FAST-NEXT:    vaddss %xmm1, %xmm4, %xmm1
991; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
992; AVX-FAST-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
993; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
994; AVX-FAST-NEXT:    vaddss %xmm4, %xmm1, %xmm1
995; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
996; AVX-FAST-NEXT:    vaddss %xmm2, %xmm1, %xmm1
997; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
998; AVX-FAST-NEXT:    vhaddps %xmm3, %xmm3, %xmm1
999; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
1000; AVX-FAST-NEXT:    vaddss %xmm2, %xmm1, %xmm1
1001; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3]
1002; AVX-FAST-NEXT:    vaddss %xmm2, %xmm1, %xmm1
1003; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
1004; AVX-FAST-NEXT:    retq
1005  %5 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %0)
1006  %6 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %1)
1007  %7 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %2)
1008  %8 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %3)
1009  %9 = insertelement <4 x float> undef, float %5, i32 0
1010  %10 = insertelement <4 x float> %9,   float %6, i32 1
1011  %11 = insertelement <4 x float> %10,  float %7, i32 2
1012  %12 = insertelement <4 x float> %11,  float %8, i32 3
1013  ret <4 x float> %12
1014}
1015declare float @llvm.vector.reduce.fadd.f32.v4f32(float, <4 x float>)
1016
1017define <4 x float> @reduction_sum_v4f32_v4f32_reassoc(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {
1018; SSSE3-SLOW-LABEL: reduction_sum_v4f32_v4f32_reassoc:
1019; SSSE3-SLOW:       # %bb.0:
1020; SSSE3-SLOW-NEXT:    movaps %xmm0, %xmm4
1021; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
1022; SSSE3-SLOW-NEXT:    addps %xmm0, %xmm4
1023; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm0 = xmm4[1,1,3,3]
1024; SSSE3-SLOW-NEXT:    movaps %xmm1, %xmm5
1025; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]
1026; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm5
1027; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm5[1,1,3,3]
1028; SSSE3-SLOW-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1029; SSSE3-SLOW-NEXT:    movaps %xmm2, %xmm1
1030; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
1031; SSSE3-SLOW-NEXT:    addps %xmm2, %xmm1
1032; SSSE3-SLOW-NEXT:    movaps %xmm3, %xmm2
1033; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1]
1034; SSSE3-SLOW-NEXT:    addps %xmm3, %xmm2
1035; SSSE3-SLOW-NEXT:    movaps %xmm2, %xmm3
1036; SSSE3-SLOW-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm1[0]
1037; SSSE3-SLOW-NEXT:    unpcklps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
1038; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,0]
1039; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
1040; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
1041; SSSE3-SLOW-NEXT:    addps %xmm0, %xmm4
1042; SSSE3-SLOW-NEXT:    movaps %xmm4, %xmm0
1043; SSSE3-SLOW-NEXT:    retq
1044;
1045; SSSE3-FAST-LABEL: reduction_sum_v4f32_v4f32_reassoc:
1046; SSSE3-FAST:       # %bb.0:
1047; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm4
1048; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
1049; SSSE3-FAST-NEXT:    addps %xmm0, %xmm4
1050; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm0
1051; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
1052; SSSE3-FAST-NEXT:    addps %xmm1, %xmm0
1053; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm4
1054; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm0
1055; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]
1056; SSSE3-FAST-NEXT:    addps %xmm2, %xmm0
1057; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm1
1058; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
1059; SSSE3-FAST-NEXT:    addps %xmm3, %xmm1
1060; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm1
1061; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,2],xmm1[2,0]
1062; SSSE3-FAST-NEXT:    movaps %xmm4, %xmm0
1063; SSSE3-FAST-NEXT:    retq
1064;
1065; AVX-SLOW-LABEL: reduction_sum_v4f32_v4f32_reassoc:
1066; AVX-SLOW:       # %bb.0:
1067; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm0[1,0]
1068; AVX-SLOW-NEXT:    vaddps %xmm4, %xmm0, %xmm0
1069; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm1[1,0]
1070; AVX-SLOW-NEXT:    vaddps %xmm4, %xmm1, %xmm1
1071; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
1072; AVX-SLOW-NEXT:    vaddps %xmm4, %xmm2, %xmm2
1073; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm3[1,0]
1074; AVX-SLOW-NEXT:    vaddps %xmm4, %xmm3, %xmm3
1075; AVX-SLOW-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[1,1],xmm2[1,1]
1076; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm5 = xmm0[1],xmm1[1],zero,zero
1077; AVX-SLOW-NEXT:    vshufps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,0]
1078; AVX-SLOW-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0]
1079; AVX-SLOW-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1080; AVX-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
1081; AVX-SLOW-NEXT:    vaddps %xmm4, %xmm0, %xmm0
1082; AVX-SLOW-NEXT:    retq
1083;
1084; AVX-FAST-LABEL: reduction_sum_v4f32_v4f32_reassoc:
1085; AVX-FAST:       # %bb.0:
1086; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm0[1,0]
1087; AVX-FAST-NEXT:    vaddps %xmm4, %xmm0, %xmm0
1088; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm1[1,0]
1089; AVX-FAST-NEXT:    vaddps %xmm4, %xmm1, %xmm1
1090; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
1091; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm2[1,0]
1092; AVX-FAST-NEXT:    vaddps %xmm1, %xmm2, %xmm1
1093; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
1094; AVX-FAST-NEXT:    vaddps %xmm2, %xmm3, %xmm2
1095; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm2, %xmm1
1096; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[2,0]
1097; AVX-FAST-NEXT:    retq
1098  %5 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %0)
1099  %6 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %1)
1100  %7 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %2)
1101  %8 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %3)
1102  %9 = insertelement <4 x float> undef, float %5, i32 0
1103  %10 = insertelement <4 x float> %9,   float %6, i32 1
1104  %11 = insertelement <4 x float> %10,  float %7, i32 2
1105  %12 = insertelement <4 x float> %11,  float %8, i32 3
1106  ret <4 x float> %12
1107}
1108
1109define <4 x i32> @reduction_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {
1110; SSSE3-SLOW-LABEL: reduction_sum_v4i32_v4i32:
1111; SSSE3-SLOW:       # %bb.0:
1112; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
1113; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm4
1114; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,1,1]
1115; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
1116; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm5
1117; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]
1118; SSSE3-SLOW-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1119; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
1120; SSSE3-SLOW-NEXT:    paddd %xmm2, %xmm1
1121; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
1122; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
1123; SSSE3-SLOW-NEXT:    paddd %xmm3, %xmm6
1124; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,1,1]
1125; SSSE3-SLOW-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
1126; SSSE3-SLOW-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1127; SSSE3-SLOW-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
1128; SSSE3-SLOW-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
1129; SSSE3-SLOW-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
1130; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm4
1131; SSSE3-SLOW-NEXT:    movdqa %xmm4, %xmm0
1132; SSSE3-SLOW-NEXT:    retq
1133;
1134; SSSE3-FAST-LABEL: reduction_sum_v4i32_v4i32:
1135; SSSE3-FAST:       # %bb.0:
1136; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
1137; SSSE3-FAST-NEXT:    paddd %xmm4, %xmm0
1138; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
1139; SSSE3-FAST-NEXT:    paddd %xmm1, %xmm4
1140; SSSE3-FAST-NEXT:    phaddd %xmm4, %xmm0
1141; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
1142; SSSE3-FAST-NEXT:    paddd %xmm2, %xmm1
1143; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
1144; SSSE3-FAST-NEXT:    paddd %xmm3, %xmm2
1145; SSSE3-FAST-NEXT:    phaddd %xmm2, %xmm1
1146; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
1147; SSSE3-FAST-NEXT:    retq
1148;
1149; AVX-SLOW-LABEL: reduction_sum_v4i32_v4i32:
1150; AVX-SLOW:       # %bb.0:
1151; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
1152; AVX-SLOW-NEXT:    vpaddd %xmm4, %xmm0, %xmm0
1153; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
1154; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
1155; AVX-SLOW-NEXT:    vpaddd %xmm5, %xmm1, %xmm1
1156; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
1157; AVX-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
1158; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
1159; AVX-SLOW-NEXT:    vpaddd %xmm5, %xmm2, %xmm2
1160; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
1161; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
1162; AVX-SLOW-NEXT:    vpaddd %xmm6, %xmm3, %xmm3
1163; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
1164; AVX-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
1165; AVX-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
1166; AVX-SLOW-NEXT:    vpaddd %xmm5, %xmm2, %xmm2
1167; AVX-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1168; AVX-SLOW-NEXT:    vpaddd %xmm4, %xmm0, %xmm0
1169; AVX-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
1170; AVX-SLOW-NEXT:    retq
1171;
1172; AVX-FAST-LABEL: reduction_sum_v4i32_v4i32:
1173; AVX-FAST:       # %bb.0:
1174; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
1175; AVX-FAST-NEXT:    vpaddd %xmm4, %xmm0, %xmm0
1176; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
1177; AVX-FAST-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
1178; AVX-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
1179; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
1180; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
1181; AVX-FAST-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
1182; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
1183; AVX-FAST-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
1184; AVX-FAST-NEXT:    vphaddd %xmm2, %xmm1, %xmm1
1185; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
1186; AVX-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1187; AVX-FAST-NEXT:    retq
1188  %5 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %0)
1189  %6 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %1)
1190  %7 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %2)
1191  %8 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %3)
1192  %9 = insertelement <4 x i32> undef, i32 %5, i32 0
1193  %10 = insertelement <4 x i32> %9,   i32 %6, i32 1
1194  %11 = insertelement <4 x i32> %10,  i32 %7, i32 2
1195  %12 = insertelement <4 x i32> %11,  i32 %8, i32 3
1196  ret <4 x i32> %12
1197}
1198declare i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32>)
1199