1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSSE3-SLOW 3; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops | FileCheck %s --check-prefixes=SSSE3-FAST 4; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX-SLOW,AVX1-SLOW 5; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops | FileCheck %s --check-prefixes=AVX-FAST,AVX1-FAST 6; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX-SLOW,AVX2-SLOW 7; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2,fast-hops | FileCheck %s --check-prefixes=AVX-FAST,AVX2-FAST 8 9; Vectorized Pairwise Sum Reductions 10; e.g. 11; inline STYPE sum(VTYPE x) { 12; return (x[0] + x[1]) + (x[2] + x[3]); 13; } 14; 15; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) { 16; return (VTYPE) { sum( A0 ), sum( A1 ), sum( A2 ), sum( A3 ) }; 17; } 18 19define <4 x float> @pair_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) { 20; SSSE3-SLOW-LABEL: pair_sum_v4f32_v4f32: 21; SSSE3-SLOW: # %bb.0: 22; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm0 23; SSSE3-SLOW-NEXT: haddps %xmm2, %xmm3 24; SSSE3-SLOW-NEXT: haddps %xmm3, %xmm0 25; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1,3,2] 26; SSSE3-SLOW-NEXT: retq 27; 28; SSSE3-FAST-LABEL: pair_sum_v4f32_v4f32: 29; SSSE3-FAST: # %bb.0: 30; SSSE3-FAST-NEXT: haddps %xmm1, %xmm0 31; SSSE3-FAST-NEXT: haddps %xmm3, %xmm2 32; SSSE3-FAST-NEXT: haddps %xmm2, %xmm0 33; SSSE3-FAST-NEXT: retq 34; 35; AVX1-SLOW-LABEL: pair_sum_v4f32_v4f32: 36; AVX1-SLOW: # %bb.0: 37; AVX1-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm0 38; AVX1-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm1 39; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm2 = xmm0[0,2],xmm1[0,1] 40; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,1] 41; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm3, %xmm1 42; AVX1-SLOW-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3] 43; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0] 44; AVX1-SLOW-NEXT: vaddps %xmm0, %xmm1, %xmm0 45; AVX1-SLOW-NEXT: retq 46; 47; AVX-FAST-LABEL: pair_sum_v4f32_v4f32: 48; AVX-FAST: # %bb.0: 49; AVX-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0 50; AVX-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm1 51; AVX-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0 52; AVX-FAST-NEXT: retq 53; 54; AVX2-SLOW-LABEL: pair_sum_v4f32_v4f32: 55; AVX2-SLOW: # %bb.0: 56; AVX2-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm0 57; AVX2-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm1 58; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm2 = xmm0[0,2],xmm1[0,3] 59; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,1] 60; AVX2-SLOW-NEXT: vhaddps %xmm3, %xmm3, %xmm1 61; AVX2-SLOW-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3] 62; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0] 63; AVX2-SLOW-NEXT: vaddps %xmm0, %xmm1, %xmm0 64; AVX2-SLOW-NEXT: retq 65 %5 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2> 66 %6 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3> 67 %7 = fadd <2 x float> %5, %6 68 %8 = shufflevector <2 x float> %7, <2 x float> poison, <2 x i32> <i32 1, i32 undef> 69 %9 = fadd <2 x float> %7, %8 70 %10 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 0, i32 2> 71 %11 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 1, i32 3> 72 %12 = fadd <2 x float> %10, %11 73 %13 = shufflevector <2 x float> %12, <2 x float> poison, <2 x i32> <i32 1, i32 undef> 74 %14 = fadd <2 x float> %12, %13 75 %15 = shufflevector <2 x float> %9, <2 x float> %14, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef> 76 %16 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 0, i32 2> 77 %17 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 1, i32 3> 78 %18 = fadd <2 x float> %16, %17 79 %19 = shufflevector <2 x float> %18, <2 x float> poison, <2 x i32> <i32 1, i32 undef> 80 %20 = fadd <2 x float> %18, %19 81 %21 = shufflevector <2 x float> %20, <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef> 82 %22 = shufflevector <4 x float> %15, <4 x float> %21, <4 x i32> <i32 0, i32 1, i32 4, i32 undef> 83 %23 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 0, i32 2> 84 %24 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 1, i32 3> 85 %25 = fadd <2 x float> %23, %24 86 %26 = shufflevector <2 x float> %25, <2 x float> poison, <2 x i32> <i32 1, i32 undef> 87 %27 = fadd <2 x float> %25, %26 88 %28 = shufflevector <2 x float> %27, <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef> 89 %29 = shufflevector <4 x float> %22, <4 x float> %28, <4 x i32> <i32 0, i32 1, i32 2, i32 4> 90 ret <4 x float> %29 91} 92 93define <4 x i32> @pair_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) { 94; SSSE3-SLOW-LABEL: pair_sum_v4i32_v4i32: 95; SSSE3-SLOW: # %bb.0: 96; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm0 97; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,3,1,3] 98; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] 99; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm0 100; SSSE3-SLOW-NEXT: phaddd %xmm2, %xmm3 101; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3] 102; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm1 103; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,0] 104; SSSE3-SLOW-NEXT: retq 105; 106; SSSE3-FAST-LABEL: pair_sum_v4i32_v4i32: 107; SSSE3-FAST: # %bb.0: 108; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm0 109; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm2 110; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm0 111; SSSE3-FAST-NEXT: retq 112; 113; AVX1-SLOW-LABEL: pair_sum_v4i32_v4i32: 114; AVX1-SLOW: # %bb.0: 115; AVX1-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm0 116; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,3,1,3] 117; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] 118; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0 119; AVX1-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm1 120; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1] 121; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 122; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] 123; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm3, %xmm1 124; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,0,0,0] 125; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm2, %xmm1 126; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7] 127; AVX1-SLOW-NEXT: retq 128; 129; AVX1-FAST-LABEL: pair_sum_v4i32_v4i32: 130; AVX1-FAST: # %bb.0: 131; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0 132; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm1 133; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0 134; AVX1-FAST-NEXT: retq 135; 136; AVX2-SLOW-LABEL: pair_sum_v4i32_v4i32: 137; AVX2-SLOW: # %bb.0: 138; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm0 139; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] 140; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0 141; AVX2-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm1 142; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1] 143; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 144; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3] 145; AVX2-SLOW-NEXT: vphaddd %xmm3, %xmm3, %xmm1 146; AVX2-SLOW-NEXT: vpbroadcastd %xmm1, %xmm2 147; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm2, %xmm1 148; AVX2-SLOW-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3] 149; AVX2-SLOW-NEXT: retq 150; 151; AVX2-FAST-LABEL: pair_sum_v4i32_v4i32: 152; AVX2-FAST: # %bb.0: 153; AVX2-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm2 154; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0 155; AVX2-FAST-NEXT: vphaddd %xmm2, %xmm0, %xmm0 156; AVX2-FAST-NEXT: retq 157 %5 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2> 158 %6 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3> 159 %7 = add <2 x i32> %5, %6 160 %8 = shufflevector <2 x i32> %7, <2 x i32> poison, <2 x i32> <i32 1, i32 undef> 161 %9 = add <2 x i32> %7, %8 162 %10 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 0, i32 2> 163 %11 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 1, i32 3> 164 %12 = add <2 x i32> %10, %11 165 %13 = shufflevector <2 x i32> %12, <2 x i32> poison, <2 x i32> <i32 1, i32 undef> 166 %14 = add <2 x i32> %12, %13 167 %15 = shufflevector <2 x i32> %9, <2 x i32> %14, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef> 168 %16 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 0, i32 2> 169 %17 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 1, i32 3> 170 %18 = add <2 x i32> %16, %17 171 %19 = shufflevector <2 x i32> %18, <2 x i32> poison, <2 x i32> <i32 1, i32 undef> 172 %20 = add <2 x i32> %18, %19 173 %21 = shufflevector <2 x i32> %20, <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef> 174 %22 = shufflevector <4 x i32> %15, <4 x i32> %21, <4 x i32> <i32 0, i32 1, i32 4, i32 undef> 175 %23 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 0, i32 2> 176 %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 1, i32 3> 177 %25 = add <2 x i32> %23, %24 178 %26 = shufflevector <2 x i32> %25, <2 x i32> poison, <2 x i32> <i32 1, i32 undef> 179 %27 = add <2 x i32> %25, %26 180 %28 = shufflevector <2 x i32> %27, <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef> 181 %29 = shufflevector <4 x i32> %22, <4 x i32> %28, <4 x i32> <i32 0, i32 1, i32 2, i32 4> 182 ret <4 x i32> %29 183} 184 185define <8 x float> @pair_sum_v8f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3, <4 x float> %4, <4 x float> %5, <4 x float> %6, <4 x float> %7) { 186; SSSE3-SLOW-LABEL: pair_sum_v8f32_v4f32: 187; SSSE3-SLOW: # %bb.0: 188; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm0 189; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm1 190; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,3],xmm0[1,3] 191; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2,1,3] 192; SSSE3-SLOW-NEXT: addps %xmm1, %xmm0 193; SSSE3-SLOW-NEXT: haddps %xmm3, %xmm2 194; SSSE3-SLOW-NEXT: movaps %xmm5, %xmm1 195; SSSE3-SLOW-NEXT: haddps %xmm4, %xmm1 196; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm2 197; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1,3,2] 198; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0] 199; SSSE3-SLOW-NEXT: haddps %xmm7, %xmm6 200; SSSE3-SLOW-NEXT: haddps %xmm5, %xmm4 201; SSSE3-SLOW-NEXT: haddps %xmm6, %xmm4 202; SSSE3-SLOW-NEXT: movaps %xmm4, %xmm1 203; SSSE3-SLOW-NEXT: retq 204; 205; SSSE3-FAST-LABEL: pair_sum_v8f32_v4f32: 206; SSSE3-FAST: # %bb.0: 207; SSSE3-FAST-NEXT: haddps %xmm1, %xmm0 208; SSSE3-FAST-NEXT: haddps %xmm0, %xmm0 209; SSSE3-FAST-NEXT: haddps %xmm3, %xmm2 210; SSSE3-FAST-NEXT: haddps %xmm5, %xmm4 211; SSSE3-FAST-NEXT: haddps %xmm4, %xmm2 212; SSSE3-FAST-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0] 213; SSSE3-FAST-NEXT: haddps %xmm7, %xmm6 214; SSSE3-FAST-NEXT: haddps %xmm6, %xmm4 215; SSSE3-FAST-NEXT: movaps %xmm4, %xmm1 216; SSSE3-FAST-NEXT: retq 217; 218; AVX1-SLOW-LABEL: pair_sum_v8f32_v4f32: 219; AVX1-SLOW: # %bb.0: 220; AVX1-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm0 221; AVX1-SLOW-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[1,3,1,3] 222; AVX1-SLOW-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[0,2,1,3] 223; AVX1-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0 224; AVX1-SLOW-NEXT: vhaddps %xmm4, %xmm4, %xmm1 225; AVX1-SLOW-NEXT: vhaddps %xmm5, %xmm5, %xmm4 226; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm2 227; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,1] 228; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0] 229; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3] 230; AVX1-SLOW-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3] 231; AVX1-SLOW-NEXT: vaddps %xmm1, %xmm3, %xmm1 232; AVX1-SLOW-NEXT: vpermilpd {{.*#+}} xmm2 = xmm1[1,0] 233; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 234; AVX1-SLOW-NEXT: vmovddup {{.*#+}} xmm1 = xmm1[0,0] 235; AVX1-SLOW-NEXT: vhaddps %xmm7, %xmm6, %xmm2 236; AVX1-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm2 237; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 238; AVX1-SLOW-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2] 239; AVX1-SLOW-NEXT: retq 240; 241; AVX1-FAST-LABEL: pair_sum_v8f32_v4f32: 242; AVX1-FAST: # %bb.0: 243; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0 244; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0 245; AVX1-FAST-NEXT: vhaddps %xmm4, %xmm4, %xmm1 246; AVX1-FAST-NEXT: vhaddps %xmm5, %xmm5, %xmm4 247; AVX1-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm2 248; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,1] 249; AVX1-FAST-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0] 250; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3] 251; AVX1-FAST-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3] 252; AVX1-FAST-NEXT: vaddps %xmm1, %xmm3, %xmm1 253; AVX1-FAST-NEXT: vpermilpd {{.*#+}} xmm2 = xmm1[1,0] 254; AVX1-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 255; AVX1-FAST-NEXT: vmovddup {{.*#+}} xmm1 = xmm1[0,0] 256; AVX1-FAST-NEXT: vhaddps %xmm7, %xmm6, %xmm2 257; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm2, %xmm2 258; AVX1-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 259; AVX1-FAST-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2] 260; AVX1-FAST-NEXT: retq 261; 262; AVX2-SLOW-LABEL: pair_sum_v8f32_v4f32: 263; AVX2-SLOW: # %bb.0: 264; AVX2-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm0 265; AVX2-SLOW-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[1,3,1,3] 266; AVX2-SLOW-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[0,2,1,3] 267; AVX2-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0 268; AVX2-SLOW-NEXT: vhaddps %xmm4, %xmm4, %xmm1 269; AVX2-SLOW-NEXT: vhaddps %xmm5, %xmm5, %xmm4 270; AVX2-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm2 271; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3] 272; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0] 273; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3] 274; AVX2-SLOW-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3] 275; AVX2-SLOW-NEXT: vaddps %xmm1, %xmm3, %xmm1 276; AVX2-SLOW-NEXT: vpermilpd {{.*#+}} xmm2 = xmm1[1,0] 277; AVX2-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 278; AVX2-SLOW-NEXT: vmovddup {{.*#+}} xmm1 = xmm1[0,0] 279; AVX2-SLOW-NEXT: vhaddps %xmm7, %xmm6, %xmm2 280; AVX2-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm2 281; AVX2-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 282; AVX2-SLOW-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2] 283; AVX2-SLOW-NEXT: retq 284; 285; AVX2-FAST-LABEL: pair_sum_v8f32_v4f32: 286; AVX2-FAST: # %bb.0: 287; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0 288; AVX2-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0 289; AVX2-FAST-NEXT: vhaddps %xmm4, %xmm4, %xmm1 290; AVX2-FAST-NEXT: vhaddps %xmm5, %xmm5, %xmm4 291; AVX2-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm2 292; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3] 293; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0] 294; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3] 295; AVX2-FAST-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3] 296; AVX2-FAST-NEXT: vaddps %xmm1, %xmm3, %xmm1 297; AVX2-FAST-NEXT: vpermilpd {{.*#+}} xmm2 = xmm1[1,0] 298; AVX2-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 299; AVX2-FAST-NEXT: vmovddup {{.*#+}} xmm1 = xmm1[0,0] 300; AVX2-FAST-NEXT: vhaddps %xmm7, %xmm6, %xmm2 301; AVX2-FAST-NEXT: vhaddps %xmm0, %xmm2, %xmm2 302; AVX2-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 303; AVX2-FAST-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2] 304; AVX2-FAST-NEXT: retq 305 %9 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2> 306 %10 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3> 307 %11 = fadd <2 x float> %9, %10 308 %12 = shufflevector <2 x float> %11, <2 x float> poison, <2 x i32> <i32 1, i32 undef> 309 %13 = fadd <2 x float> %11, %12 310 %14 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 0, i32 2> 311 %15 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 1, i32 3> 312 %16 = fadd <2 x float> %14, %15 313 %17 = shufflevector <2 x float> %16, <2 x float> poison, <2 x i32> <i32 1, i32 undef> 314 %18 = fadd <2 x float> %16, %17 315 %19 = shufflevector <2 x float> %13, <2 x float> %18, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 316 %20 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 0, i32 2> 317 %21 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 1, i32 3> 318 %22 = fadd <2 x float> %20, %21 319 %23 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 0, i32 2> 320 %24 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 1, i32 3> 321 %25 = fadd <2 x float> %23, %24 322 %26 = shufflevector <4 x float> %4, <4 x float> poison, <2 x i32> <i32 0, i32 2> 323 %27 = shufflevector <4 x float> %4, <4 x float> poison, <2 x i32> <i32 1, i32 3> 324 %28 = fadd <2 x float> %26, %27 325 %29 = shufflevector <2 x float> %28, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> 326 %30 = shufflevector <4 x float> %5, <4 x float> poison, <2 x i32> <i32 0, i32 2> 327 %31 = shufflevector <4 x float> %5, <4 x float> poison, <2 x i32> <i32 1, i32 3> 328 %32 = fadd <2 x float> %30, %31 329 %33 = shufflevector <2 x float> %32, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> 330 %34 = shufflevector <2 x float> %22, <2 x float> %25, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef> 331 %35 = shufflevector <4 x float> %34, <4 x float> %29, <4 x i32> <i32 0, i32 1, i32 4, i32 undef> 332 %36 = shufflevector <4 x float> %35, <4 x float> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 4> 333 %37 = shufflevector <2 x float> %22, <2 x float> %25, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef> 334 %38 = shufflevector <4 x float> %37, <4 x float> %29, <4 x i32> <i32 0, i32 1, i32 5, i32 undef> 335 %39 = shufflevector <4 x float> %38, <4 x float> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 5> 336 %40 = fadd <4 x float> %36, %39 337 %41 = shufflevector <4 x float> %40, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef> 338 %42 = shufflevector <8 x float> %19, <8 x float> %41, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef> 339 %43 = shufflevector <4 x float> %6, <4 x float> poison, <2 x i32> <i32 0, i32 2> 340 %44 = shufflevector <4 x float> %6, <4 x float> poison, <2 x i32> <i32 1, i32 3> 341 %45 = fadd <2 x float> %43, %44 342 %46 = shufflevector <4 x float> %7, <4 x float> poison, <2 x i32> <i32 0, i32 2> 343 %47 = shufflevector <4 x float> %7, <4 x float> poison, <2 x i32> <i32 1, i32 3> 344 %48 = fadd <2 x float> %46, %47 345 %49 = shufflevector <2 x float> %45, <2 x float> %48, <2 x i32> <i32 0, i32 2> 346 %50 = shufflevector <2 x float> %45, <2 x float> %48, <2 x i32> <i32 1, i32 3> 347 %51 = fadd <2 x float> %49, %50 348 %52 = shufflevector <2 x float> %51, <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 349 %53 = shufflevector <8 x float> %42, <8 x float> %52, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9> 350 ret <8 x float> %53 351} 352 353define <8 x i32> @pair_sum_v8i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3, <4 x i32> %4, <4 x i32> %5, <4 x i32> %6, <4 x i32> %7) { 354; SSSE3-SLOW-LABEL: pair_sum_v8i32_v4i32: 355; SSSE3-SLOW: # %bb.0: 356; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm0 357; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,3,1,3] 358; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] 359; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm0 360; SSSE3-SLOW-NEXT: phaddd %xmm3, %xmm2 361; SSSE3-SLOW-NEXT: phaddd %xmm4, %xmm5 362; SSSE3-SLOW-NEXT: phaddd %xmm5, %xmm2 363; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,1,3,2] 364; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] 365; SSSE3-SLOW-NEXT: phaddd %xmm7, %xmm6 366; SSSE3-SLOW-NEXT: phaddd %xmm6, %xmm6 367; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm6[0,1,1,1] 368; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm2[0,2] 369; SSSE3-SLOW-NEXT: retq 370; 371; SSSE3-FAST-LABEL: pair_sum_v8i32_v4i32: 372; SSSE3-FAST: # %bb.0: 373; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm0 374; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm0 375; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm2 376; SSSE3-FAST-NEXT: phaddd %xmm5, %xmm4 377; SSSE3-FAST-NEXT: phaddd %xmm4, %xmm2 378; SSSE3-FAST-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] 379; SSSE3-FAST-NEXT: phaddd %xmm6, %xmm6 380; SSSE3-FAST-NEXT: phaddd %xmm7, %xmm7 381; SSSE3-FAST-NEXT: phaddd %xmm7, %xmm6 382; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,3],xmm6[0,2] 383; SSSE3-FAST-NEXT: movaps %xmm2, %xmm1 384; SSSE3-FAST-NEXT: retq 385; 386; AVX1-SLOW-LABEL: pair_sum_v8i32_v4i32: 387; AVX1-SLOW: # %bb.0: 388; AVX1-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm0 389; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,3,1,3] 390; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] 391; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0 392; AVX1-SLOW-NEXT: vphaddd %xmm4, %xmm4, %xmm1 393; AVX1-SLOW-NEXT: vphaddd %xmm5, %xmm5, %xmm4 394; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm2, %xmm2 395; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3] 396; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0] 397; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[0,0,0,0] 398; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm5[6,7] 399; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3] 400; AVX1-SLOW-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3] 401; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm3, %xmm1 402; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3] 403; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 404; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1] 405; AVX1-SLOW-NEXT: vphaddd %xmm7, %xmm6, %xmm2 406; AVX1-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm2 407; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 408; AVX1-SLOW-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2] 409; AVX1-SLOW-NEXT: retq 410; 411; AVX1-FAST-LABEL: pair_sum_v8i32_v4i32: 412; AVX1-FAST: # %bb.0: 413; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0 414; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0 415; AVX1-FAST-NEXT: vphaddd %xmm4, %xmm4, %xmm1 416; AVX1-FAST-NEXT: vphaddd %xmm5, %xmm5, %xmm4 417; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm2 418; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3] 419; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0] 420; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[0,0,0,0] 421; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm5[6,7] 422; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3] 423; AVX1-FAST-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3] 424; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm3, %xmm1 425; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3] 426; AVX1-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 427; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1] 428; AVX1-FAST-NEXT: vphaddd %xmm7, %xmm6, %xmm2 429; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm2, %xmm2 430; AVX1-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 431; AVX1-FAST-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2] 432; AVX1-FAST-NEXT: retq 433; 434; AVX2-SLOW-LABEL: pair_sum_v8i32_v4i32: 435; AVX2-SLOW: # %bb.0: 436; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm0 437; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,3,1,3] 438; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] 439; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0 440; AVX2-SLOW-NEXT: vphaddd %xmm4, %xmm4, %xmm1 441; AVX2-SLOW-NEXT: vphaddd %xmm5, %xmm5, %xmm4 442; AVX2-SLOW-NEXT: vphaddd %xmm3, %xmm2, %xmm2 443; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3] 444; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0] 445; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3] 446; AVX2-SLOW-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3] 447; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm3, %xmm1 448; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] 449; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] 450; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 451; AVX2-SLOW-NEXT: vphaddd %xmm7, %xmm6, %xmm1 452; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm1, %xmm1 453; AVX2-SLOW-NEXT: vpbroadcastq %xmm1, %ymm1 454; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7] 455; AVX2-SLOW-NEXT: retq 456; 457; AVX2-FAST-LABEL: pair_sum_v8i32_v4i32: 458; AVX2-FAST: # %bb.0: 459; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0 460; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0 461; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm1 462; AVX2-FAST-NEXT: vphaddd %xmm4, %xmm4, %xmm4 463; AVX2-FAST-NEXT: vphaddd %xmm5, %xmm5, %xmm5 464; AVX2-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm2 465; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm4[0,3] 466; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm5[0] 467; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm2 = xmm2[1,3],xmm4[1,3] 468; AVX2-FAST-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0,1,2],xmm5[3] 469; AVX2-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2 470; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] 471; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3] 472; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 473; AVX2-FAST-NEXT: vphaddd %xmm7, %xmm6, %xmm2 474; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm2, %xmm1 475; AVX2-FAST-NEXT: vpbroadcastq %xmm1, %ymm1 476; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7] 477; AVX2-FAST-NEXT: retq 478 %9 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2> 479 %10 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3> 480 %11 = add <2 x i32> %9, %10 481 %12 = shufflevector <2 x i32> %11, <2 x i32> poison, <2 x i32> <i32 1, i32 undef> 482 %13 = add <2 x i32> %11, %12 483 %14 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 0, i32 2> 484 %15 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 1, i32 3> 485 %16 = add <2 x i32> %14, %15 486 %17 = shufflevector <2 x i32> %16, <2 x i32> poison, <2 x i32> <i32 1, i32 undef> 487 %18 = add <2 x i32> %16, %17 488 %19 = shufflevector <2 x i32> %13, <2 x i32> %18, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 489 %20 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 0, i32 2> 490 %21 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 1, i32 3> 491 %22 = add <2 x i32> %20, %21 492 %23 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 0, i32 2> 493 %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 1, i32 3> 494 %25 = add <2 x i32> %23, %24 495 %26 = shufflevector <4 x i32> %4, <4 x i32> poison, <2 x i32> <i32 0, i32 2> 496 %27 = shufflevector <4 x i32> %4, <4 x i32> poison, <2 x i32> <i32 1, i32 3> 497 %28 = add <2 x i32> %26, %27 498 %29 = shufflevector <2 x i32> %28, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> 499 %30 = shufflevector <4 x i32> %5, <4 x i32> poison, <2 x i32> <i32 0, i32 2> 500 %31 = shufflevector <4 x i32> %5, <4 x i32> poison, <2 x i32> <i32 1, i32 3> 501 %32 = add <2 x i32> %30, %31 502 %33 = shufflevector <2 x i32> %32, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> 503 %34 = shufflevector <2 x i32> %22, <2 x i32> %25, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef> 504 %35 = shufflevector <4 x i32> %34, <4 x i32> %29, <4 x i32> <i32 0, i32 1, i32 4, i32 undef> 505 %36 = shufflevector <4 x i32> %35, <4 x i32> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 4> 506 %37 = shufflevector <2 x i32> %22, <2 x i32> %25, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef> 507 %38 = shufflevector <4 x i32> %37, <4 x i32> %29, <4 x i32> <i32 0, i32 1, i32 5, i32 undef> 508 %39 = shufflevector <4 x i32> %38, <4 x i32> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 5> 509 %40 = add <4 x i32> %36, %39 510 %41 = shufflevector <4 x i32> %40, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef> 511 %42 = shufflevector <8 x i32> %19, <8 x i32> %41, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef> 512 %43 = shufflevector <4 x i32> %6, <4 x i32> poison, <2 x i32> <i32 0, i32 2> 513 %44 = shufflevector <4 x i32> %6, <4 x i32> poison, <2 x i32> <i32 1, i32 3> 514 %45 = add <2 x i32> %43, %44 515 %46 = shufflevector <4 x i32> %7, <4 x i32> poison, <2 x i32> <i32 0, i32 2> 516 %47 = shufflevector <4 x i32> %7, <4 x i32> poison, <2 x i32> <i32 1, i32 3> 517 %48 = add <2 x i32> %46, %47 518 %49 = shufflevector <2 x i32> %45, <2 x i32> %48, <2 x i32> <i32 0, i32 2> 519 %50 = shufflevector <2 x i32> %45, <2 x i32> %48, <2 x i32> <i32 1, i32 3> 520 %51 = add <2 x i32> %49, %50 521 %52 = shufflevector <2 x i32> %51, <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 522 %53 = shufflevector <8 x i32> %42, <8 x i32> %52, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9> 523 ret <8 x i32> %53 524} 525 526; Vectorized Sequential Sum Reductions 527; e.g. 528; inline STYPE sum(VTYPE x) { 529; return ((x[0] + x[1]) + x[2]) + x[3]; 530; } 531; 532; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) { 533; return (VTYPE) { sum( A0 ), sum( A1 ), sum( A2 ), sum( A3 ) }; 534; } 535 536define <4 x float> @sequential_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) { 537; SSSE3-SLOW-LABEL: sequential_sum_v4f32_v4f32: 538; SSSE3-SLOW: # %bb.0: 539; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm4 540; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm4 541; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm5 542; SSSE3-SLOW-NEXT: unpckhps {{.*#+}} xmm5 = xmm5[2],xmm1[2],xmm5[3],xmm1[3] 543; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3] 544; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm0 = xmm2[1,1,3,3] 545; SSSE3-SLOW-NEXT: addps %xmm2, %xmm0 546; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,2],xmm0[0,1] 547; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm2[2,3] 548; SSSE3-SLOW-NEXT: addps %xmm4, %xmm5 549; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,3] 550; SSSE3-SLOW-NEXT: addps %xmm5, %xmm1 551; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm0 = xmm3[1,1,3,3] 552; SSSE3-SLOW-NEXT: addps %xmm3, %xmm0 553; SSSE3-SLOW-NEXT: movaps %xmm3, %xmm2 554; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1] 555; SSSE3-SLOW-NEXT: addps %xmm0, %xmm2 556; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3] 557; SSSE3-SLOW-NEXT: addps %xmm2, %xmm3 558; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm1[2,3] 559; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,0] 560; SSSE3-SLOW-NEXT: movaps %xmm1, %xmm0 561; SSSE3-SLOW-NEXT: retq 562; 563; SSSE3-FAST-LABEL: sequential_sum_v4f32_v4f32: 564; SSSE3-FAST: # %bb.0: 565; SSSE3-FAST-NEXT: movaps %xmm0, %xmm4 566; SSSE3-FAST-NEXT: haddps %xmm1, %xmm4 567; SSSE3-FAST-NEXT: movaps %xmm0, %xmm5 568; SSSE3-FAST-NEXT: unpckhps {{.*#+}} xmm5 = xmm5[2],xmm1[2],xmm5[3],xmm1[3] 569; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3] 570; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm2[2,3] 571; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,3] 572; SSSE3-FAST-NEXT: haddps %xmm2, %xmm2 573; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,2],xmm2[0,1] 574; SSSE3-FAST-NEXT: addps %xmm4, %xmm5 575; SSSE3-FAST-NEXT: addps %xmm5, %xmm1 576; SSSE3-FAST-NEXT: movaps %xmm3, %xmm0 577; SSSE3-FAST-NEXT: haddps %xmm3, %xmm0 578; SSSE3-FAST-NEXT: movaps %xmm3, %xmm2 579; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1] 580; SSSE3-FAST-NEXT: addps %xmm0, %xmm2 581; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3] 582; SSSE3-FAST-NEXT: addps %xmm2, %xmm3 583; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm1[2,3] 584; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,0] 585; SSSE3-FAST-NEXT: movaps %xmm1, %xmm0 586; SSSE3-FAST-NEXT: retq 587; 588; AVX-SLOW-LABEL: sequential_sum_v4f32_v4f32: 589; AVX-SLOW: # %bb.0: 590; AVX-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm4 591; AVX-SLOW-NEXT: vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] 592; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0] 593; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero 594; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3] 595; AVX-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1 596; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm4[0,2],xmm1[0,1] 597; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[3,3] 598; AVX-SLOW-NEXT: vblendps {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3] 599; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3] 600; AVX-SLOW-NEXT: vaddps %xmm3, %xmm4, %xmm4 601; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0] 602; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[2] 603; AVX-SLOW-NEXT: vaddps %xmm1, %xmm2, %xmm1 604; AVX-SLOW-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3] 605; AVX-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0 606; AVX-SLOW-NEXT: retq 607; 608; AVX-FAST-LABEL: sequential_sum_v4f32_v4f32: 609; AVX-FAST: # %bb.0: 610; AVX-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm4 611; AVX-FAST-NEXT: vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] 612; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0] 613; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero 614; AVX-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm1 615; AVX-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm4[0,2],xmm1[0,1] 616; AVX-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[3,3] 617; AVX-FAST-NEXT: vblendps {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3] 618; AVX-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm4 619; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0] 620; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[2] 621; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm1 622; AVX-FAST-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3] 623; AVX-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0 624; AVX-FAST-NEXT: retq 625 %5 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 0, i32 4> 626 %6 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 1, i32 5> 627 %7 = fadd <2 x float> %5, %6 628 %8 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 2, i32 6> 629 %9 = fadd <2 x float> %8, %7 630 %10 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 3, i32 7> 631 %11 = fadd <2 x float> %10, %9 632 %12 = shufflevector <2 x float> %11, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> 633 %13 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 634 %14 = fadd <4 x float> %13, %2 635 %15 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef> 636 %16 = fadd <4 x float> %15, %14 637 %17 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef> 638 %18 = fadd <4 x float> %17, %16 639 %19 = shufflevector <4 x float> %12, <4 x float> %18, <4 x i32> <i32 0, i32 1, i32 4, i32 undef> 640 %20 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 641 %21 = fadd <4 x float> %20, %3 642 %22 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef> 643 %23 = fadd <4 x float> %22, %21 644 %24 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef> 645 %25 = fadd <4 x float> %24, %23 646 %26 = shufflevector <4 x float> %19, <4 x float> %25, <4 x i32> <i32 0, i32 1, i32 2, i32 4> 647 ret <4 x float> %26 648} 649 650define <4 x i32> @sequential_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) { 651; SSSE3-SLOW-LABEL: sequential_sum_v4i32_v4i32: 652; SSSE3-SLOW: # %bb.0: 653; SSSE3-SLOW-NEXT: movdqa %xmm0, %xmm4 654; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm4 655; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] 656; SSSE3-SLOW-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] 657; SSSE3-SLOW-NEXT: paddd %xmm0, %xmm4 658; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1] 659; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,1,0,1] 660; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm5 661; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm5 662; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1] 663; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm6 = xmm3[2,3,2,3] 664; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm6 665; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm6 666; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm5[2,3] 667; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm6[2,0] 668; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3],xmm2[3,3] 669; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,3],xmm3[2,0] 670; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm0 671; SSSE3-SLOW-NEXT: retq 672; 673; SSSE3-FAST-LABEL: sequential_sum_v4i32_v4i32: 674; SSSE3-FAST: # %bb.0: 675; SSSE3-FAST-NEXT: movdqa %xmm0, %xmm4 676; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm4 677; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] 678; SSSE3-FAST-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] 679; SSSE3-FAST-NEXT: paddd %xmm0, %xmm4 680; SSSE3-FAST-NEXT: movdqa %xmm2, %xmm1 681; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm1 682; SSSE3-FAST-NEXT: paddd %xmm2, %xmm1 683; SSSE3-FAST-NEXT: movdqa %xmm3, %xmm5 684; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm5 685; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm6 = xmm3[2,3,2,3] 686; SSSE3-FAST-NEXT: paddd %xmm5, %xmm6 687; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm1[2,3] 688; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm6[2,0] 689; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3],xmm2[3,3] 690; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,3],xmm3[2,0] 691; SSSE3-FAST-NEXT: paddd %xmm4, %xmm0 692; SSSE3-FAST-NEXT: retq 693; 694; AVX1-SLOW-LABEL: sequential_sum_v4i32_v4i32: 695; AVX1-SLOW: # %bb.0: 696; AVX1-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm4 697; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] 698; AVX1-SLOW-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] 699; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] 700; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] 701; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7] 702; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1] 703; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 704; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0] 705; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[3,3,3,3] 706; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0] 707; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm1, %xmm0 708; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0,1,2,3],xmm2[4,5,6,7] 709; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm1, %xmm0 710; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1] 711; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[0,0,0,0] 712; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,2,2,2] 713; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm2, %xmm2 714; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm2, %xmm2 715; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 716; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7] 717; AVX1-SLOW-NEXT: retq 718; 719; AVX1-FAST-LABEL: sequential_sum_v4i32_v4i32: 720; AVX1-FAST: # %bb.0: 721; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm4 722; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] 723; AVX1-FAST-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] 724; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] 725; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] 726; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7] 727; AVX1-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm1 728; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0] 729; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[3,3,3,3] 730; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0] 731; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm1, %xmm0 732; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0,1,2,3],xmm2[4,5,6,7] 733; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm1, %xmm0 734; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm1 735; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,2,2,2] 736; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0] 737; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm1 738; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1 739; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7] 740; AVX1-FAST-NEXT: retq 741; 742; AVX2-SLOW-LABEL: sequential_sum_v4i32_v4i32: 743; AVX2-SLOW: # %bb.0: 744; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm4 745; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] 746; AVX2-SLOW-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] 747; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] 748; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] 749; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3] 750; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1] 751; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 752; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0] 753; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[3,3,3,3] 754; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0] 755; AVX2-SLOW-NEXT: vpaddd %xmm0, %xmm1, %xmm0 756; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm5[0,1],xmm2[2,3] 757; AVX2-SLOW-NEXT: vpaddd %xmm0, %xmm1, %xmm0 758; AVX2-SLOW-NEXT: vpbroadcastq %xmm3, %xmm1 759; AVX2-SLOW-NEXT: vpbroadcastd %xmm3, %xmm2 760; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,2,2,2] 761; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm2, %xmm2 762; AVX2-SLOW-NEXT: vpaddd %xmm3, %xmm2, %xmm2 763; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 764; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3] 765; AVX2-SLOW-NEXT: retq 766; 767; AVX2-FAST-LABEL: sequential_sum_v4i32_v4i32: 768; AVX2-FAST: # %bb.0: 769; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm4 770; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] 771; AVX2-FAST-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] 772; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] 773; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] 774; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3] 775; AVX2-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm1 776; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0] 777; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[3,3,3,3] 778; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0] 779; AVX2-FAST-NEXT: vpaddd %xmm0, %xmm1, %xmm0 780; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm1 = xmm5[0,1],xmm2[2,3] 781; AVX2-FAST-NEXT: vpaddd %xmm0, %xmm1, %xmm0 782; AVX2-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm1 783; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,2,2,2] 784; AVX2-FAST-NEXT: vpbroadcastd %xmm1, %xmm1 785; AVX2-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm1 786; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1 787; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3] 788; AVX2-FAST-NEXT: retq 789 %5 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 0, i32 4> 790 %6 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 1, i32 5> 791 %7 = add <2 x i32> %5, %6 792 %8 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 2, i32 6> 793 %9 = add <2 x i32> %8, %7 794 %10 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 3, i32 7> 795 %11 = add <2 x i32> %10, %9 796 %12 = shufflevector <2 x i32> %11, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> 797 %13 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 798 %14 = add <4 x i32> %13, %2 799 %15 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef> 800 %16 = add <4 x i32> %15, %14 801 %17 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef> 802 %18 = add <4 x i32> %17, %16 803 %19 = shufflevector <4 x i32> %12, <4 x i32> %18, <4 x i32> <i32 0, i32 1, i32 4, i32 undef> 804 %20 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 805 %21 = add <4 x i32> %20, %3 806 %22 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef> 807 %23 = add <4 x i32> %22, %21 808 %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef> 809 %25 = add <4 x i32> %24, %23 810 %26 = shufflevector <4 x i32> %19, <4 x i32> %25, <4 x i32> <i32 0, i32 1, i32 2, i32 4> 811 ret <4 x i32> %26 812} 813 814; Vectorized Reductions 815; e.g. 816; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) { 817; return (VTYPE) { reduce( A0 ), reduce( A1 ), reduce( A2 ), reduce( A3 ) }; 818; } 819 820define <4 x float> @reduction_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) { 821; SSSE3-SLOW-LABEL: reduction_sum_v4f32_v4f32: 822; SSSE3-SLOW: # %bb.0: 823; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm4 = xmm0[1,1,3,3] 824; SSSE3-SLOW-NEXT: addss %xmm0, %xmm4 825; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm5 826; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1] 827; SSSE3-SLOW-NEXT: addss %xmm4, %xmm5 828; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3] 829; SSSE3-SLOW-NEXT: addss %xmm5, %xmm0 830; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm4 = xmm1[1,1,3,3] 831; SSSE3-SLOW-NEXT: addss %xmm1, %xmm4 832; SSSE3-SLOW-NEXT: movaps %xmm1, %xmm5 833; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1] 834; SSSE3-SLOW-NEXT: addss %xmm4, %xmm5 835; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3,3,3] 836; SSSE3-SLOW-NEXT: addss %xmm5, %xmm1 837; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] 838; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3] 839; SSSE3-SLOW-NEXT: addss %xmm2, %xmm1 840; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm4 841; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1] 842; SSSE3-SLOW-NEXT: addss %xmm1, %xmm4 843; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3,3,3] 844; SSSE3-SLOW-NEXT: addss %xmm4, %xmm2 845; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3] 846; SSSE3-SLOW-NEXT: addss %xmm3, %xmm1 847; SSSE3-SLOW-NEXT: movaps %xmm3, %xmm4 848; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1] 849; SSSE3-SLOW-NEXT: addss %xmm1, %xmm4 850; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3] 851; SSSE3-SLOW-NEXT: addss %xmm4, %xmm3 852; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] 853; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0] 854; SSSE3-SLOW-NEXT: retq 855; 856; SSSE3-FAST-LABEL: reduction_sum_v4f32_v4f32: 857; SSSE3-FAST: # %bb.0: 858; SSSE3-FAST-NEXT: movaps %xmm0, %xmm4 859; SSSE3-FAST-NEXT: haddps %xmm0, %xmm4 860; SSSE3-FAST-NEXT: movaps %xmm0, %xmm5 861; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1] 862; SSSE3-FAST-NEXT: addss %xmm4, %xmm5 863; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3] 864; SSSE3-FAST-NEXT: addss %xmm5, %xmm0 865; SSSE3-FAST-NEXT: movaps %xmm1, %xmm4 866; SSSE3-FAST-NEXT: haddps %xmm1, %xmm4 867; SSSE3-FAST-NEXT: movaps %xmm1, %xmm5 868; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1] 869; SSSE3-FAST-NEXT: addss %xmm4, %xmm5 870; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3,3,3] 871; SSSE3-FAST-NEXT: addss %xmm5, %xmm1 872; SSSE3-FAST-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] 873; SSSE3-FAST-NEXT: movaps %xmm2, %xmm1 874; SSSE3-FAST-NEXT: haddps %xmm2, %xmm1 875; SSSE3-FAST-NEXT: movaps %xmm2, %xmm4 876; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1] 877; SSSE3-FAST-NEXT: addss %xmm1, %xmm4 878; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3,3,3] 879; SSSE3-FAST-NEXT: addss %xmm4, %xmm2 880; SSSE3-FAST-NEXT: movaps %xmm3, %xmm1 881; SSSE3-FAST-NEXT: haddps %xmm3, %xmm1 882; SSSE3-FAST-NEXT: movaps %xmm3, %xmm4 883; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1] 884; SSSE3-FAST-NEXT: addss %xmm1, %xmm4 885; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3] 886; SSSE3-FAST-NEXT: addss %xmm4, %xmm3 887; SSSE3-FAST-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] 888; SSSE3-FAST-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0] 889; SSSE3-FAST-NEXT: retq 890; 891; AVX-SLOW-LABEL: reduction_sum_v4f32_v4f32: 892; AVX-SLOW: # %bb.0: 893; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3] 894; AVX-SLOW-NEXT: vaddss %xmm4, %xmm0, %xmm4 895; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm5 = xmm0[1,0] 896; AVX-SLOW-NEXT: vaddss %xmm5, %xmm4, %xmm4 897; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3] 898; AVX-SLOW-NEXT: vaddss %xmm0, %xmm4, %xmm0 899; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3] 900; AVX-SLOW-NEXT: vaddss %xmm4, %xmm1, %xmm4 901; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm5 = xmm1[1,0] 902; AVX-SLOW-NEXT: vaddss %xmm5, %xmm4, %xmm4 903; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3] 904; AVX-SLOW-NEXT: vaddss %xmm1, %xmm4, %xmm1 905; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3] 906; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3] 907; AVX-SLOW-NEXT: vaddss %xmm1, %xmm2, %xmm1 908; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm4 = xmm2[1,0] 909; AVX-SLOW-NEXT: vaddss %xmm4, %xmm1, %xmm1 910; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3] 911; AVX-SLOW-NEXT: vaddss %xmm2, %xmm1, %xmm1 912; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3] 913; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm3[1,1,3,3] 914; AVX-SLOW-NEXT: vaddss %xmm1, %xmm3, %xmm1 915; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm2 = xmm3[1,0] 916; AVX-SLOW-NEXT: vaddss %xmm2, %xmm1, %xmm1 917; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3] 918; AVX-SLOW-NEXT: vaddss %xmm2, %xmm1, %xmm1 919; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] 920; AVX-SLOW-NEXT: retq 921; 922; AVX-FAST-LABEL: reduction_sum_v4f32_v4f32: 923; AVX-FAST: # %bb.0: 924; AVX-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm4 925; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm5 = xmm0[1,0] 926; AVX-FAST-NEXT: vaddss %xmm5, %xmm4, %xmm4 927; AVX-FAST-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3] 928; AVX-FAST-NEXT: vaddss %xmm0, %xmm4, %xmm0 929; AVX-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm4 930; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm5 = xmm1[1,0] 931; AVX-FAST-NEXT: vaddss %xmm5, %xmm4, %xmm4 932; AVX-FAST-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3] 933; AVX-FAST-NEXT: vaddss %xmm1, %xmm4, %xmm1 934; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3] 935; AVX-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm1 936; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm4 = xmm2[1,0] 937; AVX-FAST-NEXT: vaddss %xmm4, %xmm1, %xmm1 938; AVX-FAST-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3] 939; AVX-FAST-NEXT: vaddss %xmm2, %xmm1, %xmm1 940; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3] 941; AVX-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm1 942; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm2 = xmm3[1,0] 943; AVX-FAST-NEXT: vaddss %xmm2, %xmm1, %xmm1 944; AVX-FAST-NEXT: vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3] 945; AVX-FAST-NEXT: vaddss %xmm2, %xmm1, %xmm1 946; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] 947; AVX-FAST-NEXT: retq 948 %5 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %0) 949 %6 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %1) 950 %7 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %2) 951 %8 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %3) 952 %9 = insertelement <4 x float> undef, float %5, i32 0 953 %10 = insertelement <4 x float> %9, float %6, i32 1 954 %11 = insertelement <4 x float> %10, float %7, i32 2 955 %12 = insertelement <4 x float> %11, float %8, i32 3 956 ret <4 x float> %12 957} 958declare float @llvm.vector.reduce.fadd.f32.v4f32(float, <4 x float>) 959 960define <4 x float> @reduction_sum_v4f32_v4f32_reassoc(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) { 961; SSSE3-SLOW-LABEL: reduction_sum_v4f32_v4f32_reassoc: 962; SSSE3-SLOW: # %bb.0: 963; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm4 964; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1] 965; SSSE3-SLOW-NEXT: addps %xmm0, %xmm4 966; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm0 = xmm4[1,1,3,3] 967; SSSE3-SLOW-NEXT: movaps %xmm1, %xmm5 968; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1] 969; SSSE3-SLOW-NEXT: addps %xmm1, %xmm5 970; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm5[1,1,3,3] 971; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] 972; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm1 973; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1] 974; SSSE3-SLOW-NEXT: addps %xmm2, %xmm1 975; SSSE3-SLOW-NEXT: movaps %xmm3, %xmm2 976; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1] 977; SSSE3-SLOW-NEXT: addps %xmm3, %xmm2 978; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm3 979; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm3 = xmm3[0],xmm1[0] 980; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1] 981; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,0] 982; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1] 983; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0] 984; SSSE3-SLOW-NEXT: addps %xmm0, %xmm4 985; SSSE3-SLOW-NEXT: movaps %xmm4, %xmm0 986; SSSE3-SLOW-NEXT: retq 987; 988; SSSE3-FAST-LABEL: reduction_sum_v4f32_v4f32_reassoc: 989; SSSE3-FAST: # %bb.0: 990; SSSE3-FAST-NEXT: movaps %xmm0, %xmm4 991; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1] 992; SSSE3-FAST-NEXT: addps %xmm0, %xmm4 993; SSSE3-FAST-NEXT: movaps %xmm1, %xmm0 994; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] 995; SSSE3-FAST-NEXT: addps %xmm1, %xmm0 996; SSSE3-FAST-NEXT: haddps %xmm0, %xmm4 997; SSSE3-FAST-NEXT: movaps %xmm2, %xmm0 998; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1] 999; SSSE3-FAST-NEXT: addps %xmm2, %xmm0 1000; SSSE3-FAST-NEXT: movaps %xmm3, %xmm1 1001; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1] 1002; SSSE3-FAST-NEXT: addps %xmm3, %xmm1 1003; SSSE3-FAST-NEXT: haddps %xmm1, %xmm0 1004; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,2],xmm0[0,2] 1005; SSSE3-FAST-NEXT: movaps %xmm4, %xmm0 1006; SSSE3-FAST-NEXT: retq 1007; 1008; AVX-SLOW-LABEL: reduction_sum_v4f32_v4f32_reassoc: 1009; AVX-SLOW: # %bb.0: 1010; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm4 = xmm0[1,0] 1011; AVX-SLOW-NEXT: vaddps %xmm4, %xmm0, %xmm0 1012; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm4 = xmm1[1,0] 1013; AVX-SLOW-NEXT: vaddps %xmm4, %xmm1, %xmm1 1014; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm4 = xmm2[1,0] 1015; AVX-SLOW-NEXT: vaddps %xmm4, %xmm2, %xmm2 1016; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm4 = xmm3[1,0] 1017; AVX-SLOW-NEXT: vaddps %xmm4, %xmm3, %xmm3 1018; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm4 = xmm3[1,1],xmm2[1,1] 1019; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm5 = xmm0[1],xmm1[1],zero,zero 1020; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,0] 1021; AVX-SLOW-NEXT: vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0] 1022; AVX-SLOW-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] 1023; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0] 1024; AVX-SLOW-NEXT: vaddps %xmm4, %xmm0, %xmm0 1025; AVX-SLOW-NEXT: retq 1026; 1027; AVX-FAST-LABEL: reduction_sum_v4f32_v4f32_reassoc: 1028; AVX-FAST: # %bb.0: 1029; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm4 = xmm0[1,0] 1030; AVX-FAST-NEXT: vaddps %xmm4, %xmm0, %xmm0 1031; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm4 = xmm1[1,0] 1032; AVX-FAST-NEXT: vaddps %xmm4, %xmm1, %xmm1 1033; AVX-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0 1034; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm1 = xmm2[1,0] 1035; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm1 1036; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm2 = xmm3[1,0] 1037; AVX-FAST-NEXT: vaddps %xmm2, %xmm3, %xmm2 1038; AVX-FAST-NEXT: vhaddps %xmm2, %xmm1, %xmm1 1039; AVX-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2] 1040; AVX-FAST-NEXT: retq 1041 %5 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %0) 1042 %6 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %1) 1043 %7 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %2) 1044 %8 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %3) 1045 %9 = insertelement <4 x float> undef, float %5, i32 0 1046 %10 = insertelement <4 x float> %9, float %6, i32 1 1047 %11 = insertelement <4 x float> %10, float %7, i32 2 1048 %12 = insertelement <4 x float> %11, float %8, i32 3 1049 ret <4 x float> %12 1050} 1051 1052define <4 x i32> @reduction_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) { 1053; SSSE3-SLOW-LABEL: reduction_sum_v4i32_v4i32: 1054; SSSE3-SLOW: # %bb.0: 1055; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3] 1056; SSSE3-SLOW-NEXT: paddd %xmm0, %xmm4 1057; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,1,1] 1058; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3] 1059; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm5 1060; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm5[1,1,1,1] 1061; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] 1062; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3] 1063; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm1 1064; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1] 1065; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm6 = xmm3[2,3,2,3] 1066; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm6 1067; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm3 = xmm6[1,1,1,1] 1068; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] 1069; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] 1070; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1] 1071; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1] 1072; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0] 1073; SSSE3-SLOW-NEXT: paddd %xmm0, %xmm4 1074; SSSE3-SLOW-NEXT: movdqa %xmm4, %xmm0 1075; SSSE3-SLOW-NEXT: retq 1076; 1077; SSSE3-FAST-LABEL: reduction_sum_v4i32_v4i32: 1078; SSSE3-FAST: # %bb.0: 1079; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3] 1080; SSSE3-FAST-NEXT: paddd %xmm4, %xmm0 1081; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3] 1082; SSSE3-FAST-NEXT: paddd %xmm1, %xmm4 1083; SSSE3-FAST-NEXT: phaddd %xmm4, %xmm0 1084; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3] 1085; SSSE3-FAST-NEXT: paddd %xmm2, %xmm1 1086; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3] 1087; SSSE3-FAST-NEXT: paddd %xmm3, %xmm2 1088; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm1 1089; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2] 1090; SSSE3-FAST-NEXT: retq 1091; 1092; AVX-SLOW-LABEL: reduction_sum_v4i32_v4i32: 1093; AVX-SLOW: # %bb.0: 1094; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3] 1095; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0 1096; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1] 1097; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3] 1098; AVX-SLOW-NEXT: vpaddd %xmm5, %xmm1, %xmm1 1099; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1] 1100; AVX-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1] 1101; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3] 1102; AVX-SLOW-NEXT: vpaddd %xmm5, %xmm2, %xmm2 1103; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1] 1104; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[2,3,2,3] 1105; AVX-SLOW-NEXT: vpaddd %xmm6, %xmm3, %xmm3 1106; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1] 1107; AVX-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1] 1108; AVX-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] 1109; AVX-SLOW-NEXT: vpaddd %xmm5, %xmm2, %xmm2 1110; AVX-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] 1111; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0 1112; AVX-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] 1113; AVX-SLOW-NEXT: retq 1114; 1115; AVX-FAST-LABEL: reduction_sum_v4i32_v4i32: 1116; AVX-FAST: # %bb.0: 1117; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3] 1118; AVX-FAST-NEXT: vpaddd %xmm4, %xmm0, %xmm0 1119; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3] 1120; AVX-FAST-NEXT: vpaddd %xmm4, %xmm1, %xmm1 1121; AVX-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0 1122; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3] 1123; AVX-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1 1124; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3] 1125; AVX-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2 1126; AVX-FAST-NEXT: vphaddd %xmm2, %xmm1, %xmm1 1127; AVX-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2] 1128; AVX-FAST-NEXT: retq 1129 %5 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %0) 1130 %6 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %1) 1131 %7 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %2) 1132 %8 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %3) 1133 %9 = insertelement <4 x i32> undef, i32 %5, i32 0 1134 %10 = insertelement <4 x i32> %9, i32 %6, i32 1 1135 %11 = insertelement <4 x i32> %10, i32 %7, i32 2 1136 %12 = insertelement <4 x i32> %11, i32 %8, i32 3 1137 ret <4 x i32> %12 1138} 1139declare i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32>) 1140