1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE2 3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1 4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX256,AVX2 5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX256,AVX512,AVX512F 6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,AVX256,AVX512,AVX512BW 7 8define i32 @_Z10test_shortPsS_i_128(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 { 9; SSE2-LABEL: _Z10test_shortPsS_i_128: 10; SSE2: # %bb.0: # %entry 11; SSE2-NEXT: movl %edx, %eax 12; SSE2-NEXT: pxor %xmm0, %xmm0 13; SSE2-NEXT: xorl %ecx, %ecx 14; SSE2-NEXT: .p2align 4, 0x90 15; SSE2-NEXT: .LBB0_1: # %vector.body 16; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 17; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 18; SSE2-NEXT: movq {{.*#+}} xmm2 = mem[0],zero 19; SSE2-NEXT: movdqa %xmm2, %xmm3 20; SSE2-NEXT: pmulhw %xmm1, %xmm3 21; SSE2-NEXT: pmullw %xmm1, %xmm2 22; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] 23; SSE2-NEXT: paddd %xmm2, %xmm0 24; SSE2-NEXT: addq $8, %rcx 25; SSE2-NEXT: cmpq %rcx, %rax 26; SSE2-NEXT: jne .LBB0_1 27; SSE2-NEXT: # %bb.2: # %middle.block 28; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 29; SSE2-NEXT: paddd %xmm0, %xmm1 30; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3] 31; SSE2-NEXT: paddd %xmm1, %xmm0 32; SSE2-NEXT: movd %xmm0, %eax 33; SSE2-NEXT: retq 34; 35; AVX-LABEL: _Z10test_shortPsS_i_128: 36; AVX: # %bb.0: # %entry 37; AVX-NEXT: movl %edx, %eax 38; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0 39; AVX-NEXT: xorl %ecx, %ecx 40; AVX-NEXT: .p2align 4, 0x90 41; AVX-NEXT: .LBB0_1: # %vector.body 42; AVX-NEXT: # =>This Inner Loop Header: Depth=1 43; AVX-NEXT: vpmovsxwd (%rdi,%rcx,2), %xmm1 44; AVX-NEXT: vpmovsxwd (%rsi,%rcx,2), %xmm2 45; AVX-NEXT: vpmulld %xmm1, %xmm2, %xmm1 46; AVX-NEXT: vpaddd %xmm0, %xmm1, %xmm0 47; AVX-NEXT: addq $8, %rcx 48; AVX-NEXT: cmpq %rcx, %rax 49; AVX-NEXT: jne .LBB0_1 50; AVX-NEXT: # %bb.2: # %middle.block 51; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 52; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 53; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 54; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 55; AVX-NEXT: vmovd %xmm0, %eax 56; AVX-NEXT: retq 57entry: 58 %3 = zext i32 %2 to i64 59 br label %vector.body 60 61vector.body: 62 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 63 %vec.phi = phi <4 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] 64 %4 = getelementptr inbounds i16, i16* %0, i64 %index 65 %5 = bitcast i16* %4 to <4 x i16>* 66 %wide.load = load <4 x i16>, <4 x i16>* %5, align 2 67 %6 = sext <4 x i16> %wide.load to <4 x i32> 68 %7 = getelementptr inbounds i16, i16* %1, i64 %index 69 %8 = bitcast i16* %7 to <4 x i16>* 70 %wide.load14 = load <4 x i16>, <4 x i16>* %8, align 2 71 %9 = sext <4 x i16> %wide.load14 to <4 x i32> 72 %10 = mul nsw <4 x i32> %9, %6 73 %11 = add nsw <4 x i32> %10, %vec.phi 74 %index.next = add i64 %index, 8 75 %12 = icmp eq i64 %index.next, %3 76 br i1 %12, label %middle.block, label %vector.body 77 78middle.block: 79 %rdx.shuf15 = shufflevector <4 x i32> %11, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> 80 %bin.rdx16 = add <4 x i32> %11, %rdx.shuf15 81 %rdx.shuf17 = shufflevector <4 x i32> %bin.rdx16, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 82 %bin.rdx18 = add <4 x i32> %bin.rdx16, %rdx.shuf17 83 %13 = extractelement <4 x i32> %bin.rdx18, i32 0 84 ret i32 %13 85} 86 87define i32 @_Z10test_shortPsS_i_256(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 { 88; SSE2-LABEL: _Z10test_shortPsS_i_256: 89; SSE2: # %bb.0: # %entry 90; SSE2-NEXT: movl %edx, %eax 91; SSE2-NEXT: pxor %xmm0, %xmm0 92; SSE2-NEXT: xorl %ecx, %ecx 93; SSE2-NEXT: pxor %xmm1, %xmm1 94; SSE2-NEXT: .p2align 4, 0x90 95; SSE2-NEXT: .LBB1_1: # %vector.body 96; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 97; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm2 98; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm3 99; SSE2-NEXT: pmaddwd %xmm2, %xmm3 100; SSE2-NEXT: paddd %xmm3, %xmm1 101; SSE2-NEXT: addq $8, %rcx 102; SSE2-NEXT: cmpq %rcx, %rax 103; SSE2-NEXT: jne .LBB1_1 104; SSE2-NEXT: # %bb.2: # %middle.block 105; SSE2-NEXT: paddd %xmm0, %xmm1 106; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] 107; SSE2-NEXT: paddd %xmm1, %xmm0 108; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 109; SSE2-NEXT: paddd %xmm0, %xmm1 110; SSE2-NEXT: movd %xmm1, %eax 111; SSE2-NEXT: retq 112; 113; AVX1-LABEL: _Z10test_shortPsS_i_256: 114; AVX1: # %bb.0: # %entry 115; AVX1-NEXT: movl %edx, %eax 116; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0 117; AVX1-NEXT: xorl %ecx, %ecx 118; AVX1-NEXT: .p2align 4, 0x90 119; AVX1-NEXT: .LBB1_1: # %vector.body 120; AVX1-NEXT: # =>This Inner Loop Header: Depth=1 121; AVX1-NEXT: vmovdqu (%rsi,%rcx,2), %xmm1 122; AVX1-NEXT: vpmaddwd (%rdi,%rcx,2), %xmm1, %xmm1 123; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm1 124; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] 125; AVX1-NEXT: addq $8, %rcx 126; AVX1-NEXT: cmpq %rcx, %rax 127; AVX1-NEXT: jne .LBB1_1 128; AVX1-NEXT: # %bb.2: # %middle.block 129; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 130; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 131; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 132; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 133; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 134; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 135; AVX1-NEXT: vmovd %xmm0, %eax 136; AVX1-NEXT: vzeroupper 137; AVX1-NEXT: retq 138; 139; AVX256-LABEL: _Z10test_shortPsS_i_256: 140; AVX256: # %bb.0: # %entry 141; AVX256-NEXT: movl %edx, %eax 142; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0 143; AVX256-NEXT: xorl %ecx, %ecx 144; AVX256-NEXT: .p2align 4, 0x90 145; AVX256-NEXT: .LBB1_1: # %vector.body 146; AVX256-NEXT: # =>This Inner Loop Header: Depth=1 147; AVX256-NEXT: vmovdqu (%rsi,%rcx,2), %xmm1 148; AVX256-NEXT: vpmaddwd (%rdi,%rcx,2), %xmm1, %xmm1 149; AVX256-NEXT: vpaddd %ymm0, %ymm1, %ymm0 150; AVX256-NEXT: addq $8, %rcx 151; AVX256-NEXT: cmpq %rcx, %rax 152; AVX256-NEXT: jne .LBB1_1 153; AVX256-NEXT: # %bb.2: # %middle.block 154; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1 155; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0 156; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 157; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0 158; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 159; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0 160; AVX256-NEXT: vmovd %xmm0, %eax 161; AVX256-NEXT: vzeroupper 162; AVX256-NEXT: retq 163entry: 164 %3 = zext i32 %2 to i64 165 br label %vector.body 166 167vector.body: 168 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 169 %vec.phi = phi <8 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] 170 %4 = getelementptr inbounds i16, i16* %0, i64 %index 171 %5 = bitcast i16* %4 to <8 x i16>* 172 %wide.load = load <8 x i16>, <8 x i16>* %5, align 2 173 %6 = sext <8 x i16> %wide.load to <8 x i32> 174 %7 = getelementptr inbounds i16, i16* %1, i64 %index 175 %8 = bitcast i16* %7 to <8 x i16>* 176 %wide.load14 = load <8 x i16>, <8 x i16>* %8, align 2 177 %9 = sext <8 x i16> %wide.load14 to <8 x i32> 178 %10 = mul nsw <8 x i32> %9, %6 179 %11 = add nsw <8 x i32> %10, %vec.phi 180 %index.next = add i64 %index, 8 181 %12 = icmp eq i64 %index.next, %3 182 br i1 %12, label %middle.block, label %vector.body 183 184middle.block: 185 %rdx.shuf = shufflevector <8 x i32> %11, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef> 186 %bin.rdx = add <8 x i32> %11, %rdx.shuf 187 %rdx.shuf15 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 188 %bin.rdx16 = add <8 x i32> %bin.rdx, %rdx.shuf15 189 %rdx.shuf17 = shufflevector <8 x i32> %bin.rdx16, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 190 %bin.rdx18 = add <8 x i32> %bin.rdx16, %rdx.shuf17 191 %13 = extractelement <8 x i32> %bin.rdx18, i32 0 192 ret i32 %13 193} 194 195define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 { 196; SSE2-LABEL: _Z10test_shortPsS_i_512: 197; SSE2: # %bb.0: # %entry 198; SSE2-NEXT: movl %edx, %eax 199; SSE2-NEXT: pxor %xmm0, %xmm0 200; SSE2-NEXT: xorl %ecx, %ecx 201; SSE2-NEXT: pxor %xmm2, %xmm2 202; SSE2-NEXT: pxor %xmm1, %xmm1 203; SSE2-NEXT: .p2align 4, 0x90 204; SSE2-NEXT: .LBB2_1: # %vector.body 205; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 206; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm3 207; SSE2-NEXT: movdqu 16(%rdi,%rcx,2), %xmm4 208; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm5 209; SSE2-NEXT: pmaddwd %xmm3, %xmm5 210; SSE2-NEXT: paddd %xmm5, %xmm2 211; SSE2-NEXT: movdqu 16(%rsi,%rcx,2), %xmm3 212; SSE2-NEXT: pmaddwd %xmm4, %xmm3 213; SSE2-NEXT: paddd %xmm3, %xmm1 214; SSE2-NEXT: addq $16, %rcx 215; SSE2-NEXT: cmpq %rcx, %rax 216; SSE2-NEXT: jne .LBB2_1 217; SSE2-NEXT: # %bb.2: # %middle.block 218; SSE2-NEXT: paddd %xmm0, %xmm2 219; SSE2-NEXT: paddd %xmm0, %xmm1 220; SSE2-NEXT: paddd %xmm2, %xmm1 221; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] 222; SSE2-NEXT: paddd %xmm1, %xmm0 223; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 224; SSE2-NEXT: paddd %xmm0, %xmm1 225; SSE2-NEXT: movd %xmm1, %eax 226; SSE2-NEXT: retq 227; 228; AVX1-LABEL: _Z10test_shortPsS_i_512: 229; AVX1: # %bb.0: # %entry 230; AVX1-NEXT: movl %edx, %eax 231; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0 232; AVX1-NEXT: xorl %ecx, %ecx 233; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 234; AVX1-NEXT: .p2align 4, 0x90 235; AVX1-NEXT: .LBB2_1: # %vector.body 236; AVX1-NEXT: # =>This Inner Loop Header: Depth=1 237; AVX1-NEXT: vmovdqu (%rsi,%rcx,2), %xmm2 238; AVX1-NEXT: vmovdqu 16(%rsi,%rcx,2), %xmm3 239; AVX1-NEXT: vpmaddwd 16(%rdi,%rcx,2), %xmm3, %xmm3 240; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 241; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3 242; AVX1-NEXT: vpmaddwd (%rdi,%rcx,2), %xmm2, %xmm2 243; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm1 244; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 245; AVX1-NEXT: addq $16, %rcx 246; AVX1-NEXT: cmpq %rcx, %rax 247; AVX1-NEXT: jne .LBB2_1 248; AVX1-NEXT: # %bb.2: # %middle.block 249; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 250; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 251; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2 252; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0 253; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0 254; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 255; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 256; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 257; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 258; AVX1-NEXT: vmovd %xmm0, %eax 259; AVX1-NEXT: vzeroupper 260; AVX1-NEXT: retq 261; 262; AVX2-LABEL: _Z10test_shortPsS_i_512: 263; AVX2: # %bb.0: # %entry 264; AVX2-NEXT: movl %edx, %eax 265; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0 266; AVX2-NEXT: xorl %ecx, %ecx 267; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 268; AVX2-NEXT: .p2align 4, 0x90 269; AVX2-NEXT: .LBB2_1: # %vector.body 270; AVX2-NEXT: # =>This Inner Loop Header: Depth=1 271; AVX2-NEXT: vmovdqu (%rsi,%rcx,2), %ymm2 272; AVX2-NEXT: vpmaddwd (%rdi,%rcx,2), %ymm2, %ymm2 273; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1 274; AVX2-NEXT: addq $16, %rcx 275; AVX2-NEXT: cmpq %rcx, %rax 276; AVX2-NEXT: jne .LBB2_1 277; AVX2-NEXT: # %bb.2: # %middle.block 278; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0 279; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 280; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 281; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 282; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 283; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 284; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 285; AVX2-NEXT: vmovd %xmm0, %eax 286; AVX2-NEXT: vzeroupper 287; AVX2-NEXT: retq 288; 289; AVX512-LABEL: _Z10test_shortPsS_i_512: 290; AVX512: # %bb.0: # %entry 291; AVX512-NEXT: movl %edx, %eax 292; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0 293; AVX512-NEXT: xorl %ecx, %ecx 294; AVX512-NEXT: .p2align 4, 0x90 295; AVX512-NEXT: .LBB2_1: # %vector.body 296; AVX512-NEXT: # =>This Inner Loop Header: Depth=1 297; AVX512-NEXT: vmovdqu (%rsi,%rcx,2), %ymm1 298; AVX512-NEXT: vpmaddwd (%rdi,%rcx,2), %ymm1, %ymm1 299; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0 300; AVX512-NEXT: addq $16, %rcx 301; AVX512-NEXT: cmpq %rcx, %rax 302; AVX512-NEXT: jne .LBB2_1 303; AVX512-NEXT: # %bb.2: # %middle.block 304; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 305; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0 306; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 307; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 308; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 309; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 310; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 311; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 312; AVX512-NEXT: vmovd %xmm0, %eax 313; AVX512-NEXT: vzeroupper 314; AVX512-NEXT: retq 315entry: 316 %3 = zext i32 %2 to i64 317 br label %vector.body 318 319vector.body: 320 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 321 %vec.phi = phi <16 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] 322 %4 = getelementptr inbounds i16, i16* %0, i64 %index 323 %5 = bitcast i16* %4 to <16 x i16>* 324 %wide.load = load <16 x i16>, <16 x i16>* %5, align 2 325 %6 = sext <16 x i16> %wide.load to <16 x i32> 326 %7 = getelementptr inbounds i16, i16* %1, i64 %index 327 %8 = bitcast i16* %7 to <16 x i16>* 328 %wide.load14 = load <16 x i16>, <16 x i16>* %8, align 2 329 %9 = sext <16 x i16> %wide.load14 to <16 x i32> 330 %10 = mul nsw <16 x i32> %9, %6 331 %11 = add nsw <16 x i32> %10, %vec.phi 332 %index.next = add i64 %index, 16 333 %12 = icmp eq i64 %index.next, %3 334 br i1 %12, label %middle.block, label %vector.body 335 336middle.block: 337 %rdx.shuf1 = shufflevector <16 x i32> %11, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 338 %bin.rdx1 = add <16 x i32> %11, %rdx.shuf1 339 %rdx.shuf = shufflevector <16 x i32> %bin.rdx1, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 340 %bin.rdx = add <16 x i32> %bin.rdx1, %rdx.shuf 341 %rdx.shuf15 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 342 %bin.rdx16 = add <16 x i32> %bin.rdx, %rdx.shuf15 343 %rdx.shuf17 = shufflevector <16 x i32> %bin.rdx16, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 344 %bin.rdx18 = add <16 x i32> %bin.rdx16, %rdx.shuf17 345 %13 = extractelement <16 x i32> %bin.rdx18, i32 0 346 ret i32 %13 347} 348 349define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 { 350; SSE2-LABEL: _Z10test_shortPsS_i_1024: 351; SSE2: # %bb.0: # %entry 352; SSE2-NEXT: movl %edx, %eax 353; SSE2-NEXT: pxor %xmm8, %xmm8 354; SSE2-NEXT: xorl %ecx, %ecx 355; SSE2-NEXT: pxor %xmm2, %xmm2 356; SSE2-NEXT: pxor %xmm4, %xmm4 357; SSE2-NEXT: pxor %xmm1, %xmm1 358; SSE2-NEXT: pxor %xmm3, %xmm3 359; SSE2-NEXT: .p2align 4, 0x90 360; SSE2-NEXT: .LBB3_1: # %vector.body 361; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 362; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm5 363; SSE2-NEXT: movdqu 16(%rdi,%rcx,2), %xmm6 364; SSE2-NEXT: movdqu 32(%rdi,%rcx,2), %xmm7 365; SSE2-NEXT: movdqu 48(%rdi,%rcx,2), %xmm9 366; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm0 367; SSE2-NEXT: pmaddwd %xmm5, %xmm0 368; SSE2-NEXT: paddd %xmm0, %xmm2 369; SSE2-NEXT: movdqu 16(%rsi,%rcx,2), %xmm0 370; SSE2-NEXT: pmaddwd %xmm6, %xmm0 371; SSE2-NEXT: paddd %xmm0, %xmm4 372; SSE2-NEXT: movdqu 32(%rsi,%rcx,2), %xmm0 373; SSE2-NEXT: pmaddwd %xmm7, %xmm0 374; SSE2-NEXT: paddd %xmm0, %xmm1 375; SSE2-NEXT: movdqu 48(%rsi,%rcx,2), %xmm0 376; SSE2-NEXT: pmaddwd %xmm9, %xmm0 377; SSE2-NEXT: paddd %xmm0, %xmm3 378; SSE2-NEXT: addq $16, %rcx 379; SSE2-NEXT: cmpq %rcx, %rax 380; SSE2-NEXT: jne .LBB3_1 381; SSE2-NEXT: # %bb.2: # %middle.block 382; SSE2-NEXT: paddd %xmm8, %xmm4 383; SSE2-NEXT: paddd %xmm8, %xmm3 384; SSE2-NEXT: paddd %xmm4, %xmm3 385; SSE2-NEXT: paddd %xmm8, %xmm2 386; SSE2-NEXT: paddd %xmm8, %xmm1 387; SSE2-NEXT: paddd %xmm3, %xmm1 388; SSE2-NEXT: paddd %xmm2, %xmm1 389; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] 390; SSE2-NEXT: paddd %xmm1, %xmm0 391; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 392; SSE2-NEXT: paddd %xmm0, %xmm1 393; SSE2-NEXT: movd %xmm1, %eax 394; SSE2-NEXT: retq 395; 396; AVX1-LABEL: _Z10test_shortPsS_i_1024: 397; AVX1: # %bb.0: # %entry 398; AVX1-NEXT: movl %edx, %eax 399; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0 400; AVX1-NEXT: xorl %ecx, %ecx 401; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 402; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 403; AVX1-NEXT: .p2align 4, 0x90 404; AVX1-NEXT: .LBB3_1: # %vector.body 405; AVX1-NEXT: # =>This Inner Loop Header: Depth=1 406; AVX1-NEXT: vmovdqu (%rsi,%rcx,2), %xmm3 407; AVX1-NEXT: vmovdqu 16(%rsi,%rcx,2), %xmm4 408; AVX1-NEXT: vmovdqu 32(%rsi,%rcx,2), %xmm5 409; AVX1-NEXT: vmovdqu 48(%rsi,%rcx,2), %xmm6 410; AVX1-NEXT: vpmaddwd 48(%rdi,%rcx,2), %xmm6, %xmm6 411; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm7 412; AVX1-NEXT: vpaddd %xmm7, %xmm6, %xmm6 413; AVX1-NEXT: vpmaddwd 32(%rdi,%rcx,2), %xmm5, %xmm5 414; AVX1-NEXT: vpaddd %xmm2, %xmm5, %xmm2 415; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm2, %ymm2 416; AVX1-NEXT: vpmaddwd 16(%rdi,%rcx,2), %xmm4, %xmm4 417; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5 418; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4 419; AVX1-NEXT: vpmaddwd (%rdi,%rcx,2), %xmm3, %xmm3 420; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1 421; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1 422; AVX1-NEXT: addq $16, %rcx 423; AVX1-NEXT: cmpq %rcx, %rax 424; AVX1-NEXT: jne .LBB3_1 425; AVX1-NEXT: # %bb.2: # %middle.block 426; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm3 427; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 428; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 429; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2 430; AVX1-NEXT: vpaddd %xmm5, %xmm2, %xmm2 431; AVX1-NEXT: vpaddd %xmm2, %xmm5, %xmm2 432; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm2 433; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0 434; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0 435; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0 436; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 437; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 438; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 439; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 440; AVX1-NEXT: vmovd %xmm0, %eax 441; AVX1-NEXT: vzeroupper 442; AVX1-NEXT: retq 443; 444; AVX2-LABEL: _Z10test_shortPsS_i_1024: 445; AVX2: # %bb.0: # %entry 446; AVX2-NEXT: movl %edx, %eax 447; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0 448; AVX2-NEXT: xorl %ecx, %ecx 449; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 450; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 451; AVX2-NEXT: .p2align 4, 0x90 452; AVX2-NEXT: .LBB3_1: # %vector.body 453; AVX2-NEXT: # =>This Inner Loop Header: Depth=1 454; AVX2-NEXT: vmovdqu (%rsi,%rcx,2), %ymm3 455; AVX2-NEXT: vmovdqu 32(%rsi,%rcx,2), %ymm4 456; AVX2-NEXT: vpmaddwd 32(%rdi,%rcx,2), %ymm4, %ymm4 457; AVX2-NEXT: vpaddd %ymm2, %ymm4, %ymm2 458; AVX2-NEXT: vpmaddwd (%rdi,%rcx,2), %ymm3, %ymm3 459; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1 460; AVX2-NEXT: addq $16, %rcx 461; AVX2-NEXT: cmpq %rcx, %rax 462; AVX2-NEXT: jne .LBB3_1 463; AVX2-NEXT: # %bb.2: # %middle.block 464; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm1 465; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0 466; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0 467; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 468; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 469; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 470; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 471; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 472; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 473; AVX2-NEXT: vmovd %xmm0, %eax 474; AVX2-NEXT: vzeroupper 475; AVX2-NEXT: retq 476; 477; AVX512F-LABEL: _Z10test_shortPsS_i_1024: 478; AVX512F: # %bb.0: # %entry 479; AVX512F-NEXT: movl %edx, %eax 480; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0 481; AVX512F-NEXT: xorl %ecx, %ecx 482; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1 483; AVX512F-NEXT: .p2align 4, 0x90 484; AVX512F-NEXT: .LBB3_1: # %vector.body 485; AVX512F-NEXT: # =>This Inner Loop Header: Depth=1 486; AVX512F-NEXT: vmovdqu (%rsi,%rcx,2), %ymm2 487; AVX512F-NEXT: vmovdqu 32(%rsi,%rcx,2), %ymm3 488; AVX512F-NEXT: vpmaddwd 32(%rdi,%rcx,2), %ymm3, %ymm3 489; AVX512F-NEXT: vpmaddwd (%rdi,%rcx,2), %ymm2, %ymm2 490; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 491; AVX512F-NEXT: vpaddd %zmm1, %zmm2, %zmm1 492; AVX512F-NEXT: addq $16, %rcx 493; AVX512F-NEXT: cmpq %rcx, %rax 494; AVX512F-NEXT: jne .LBB3_1 495; AVX512F-NEXT: # %bb.2: # %middle.block 496; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0 497; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1 498; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm0 499; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1 500; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0 501; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 502; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0 503; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 504; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0 505; AVX512F-NEXT: vmovd %xmm0, %eax 506; AVX512F-NEXT: vzeroupper 507; AVX512F-NEXT: retq 508; 509; AVX512BW-LABEL: _Z10test_shortPsS_i_1024: 510; AVX512BW: # %bb.0: # %entry 511; AVX512BW-NEXT: movl %edx, %eax 512; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0 513; AVX512BW-NEXT: xorl %ecx, %ecx 514; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1 515; AVX512BW-NEXT: .p2align 4, 0x90 516; AVX512BW-NEXT: .LBB3_1: # %vector.body 517; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1 518; AVX512BW-NEXT: vmovdqu64 (%rsi,%rcx,2), %zmm2 519; AVX512BW-NEXT: vpmaddwd (%rdi,%rcx,2), %zmm2, %zmm2 520; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1 521; AVX512BW-NEXT: addq $16, %rcx 522; AVX512BW-NEXT: cmpq %rcx, %rax 523; AVX512BW-NEXT: jne .LBB3_1 524; AVX512BW-NEXT: # %bb.2: # %middle.block 525; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0 526; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1 527; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0 528; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1 529; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0 530; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 531; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0 532; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 533; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0 534; AVX512BW-NEXT: vmovd %xmm0, %eax 535; AVX512BW-NEXT: vzeroupper 536; AVX512BW-NEXT: retq 537entry: 538 %3 = zext i32 %2 to i64 539 br label %vector.body 540 541vector.body: 542 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 543 %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] 544 %4 = getelementptr inbounds i16, i16* %0, i64 %index 545 %5 = bitcast i16* %4 to <32 x i16>* 546 %wide.load = load <32 x i16>, <32 x i16>* %5, align 2 547 %6 = sext <32 x i16> %wide.load to <32 x i32> 548 %7 = getelementptr inbounds i16, i16* %1, i64 %index 549 %8 = bitcast i16* %7 to <32 x i16>* 550 %wide.load14 = load <32 x i16>, <32 x i16>* %8, align 2 551 %9 = sext <32 x i16> %wide.load14 to <32 x i32> 552 %10 = mul nsw <32 x i32> %9, %6 553 %11 = add nsw <32 x i32> %10, %vec.phi 554 %index.next = add i64 %index, 16 555 %12 = icmp eq i64 %index.next, %3 556 br i1 %12, label %middle.block, label %vector.body 557 558middle.block: 559 %rdx.shuf2 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 560 %bin.rdx2 = add <32 x i32> %11, %rdx.shuf2 561 %rdx.shuf1 = shufflevector <32 x i32> %bin.rdx2, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 562 %bin.rdx1 = add <32 x i32> %bin.rdx2, %rdx.shuf1 563 %rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 564 %bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf 565 %rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 566 %bin.rdx16 = add <32 x i32> %bin.rdx, %rdx.shuf15 567 %rdx.shuf17 = shufflevector <32 x i32> %bin.rdx16, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 568 %bin.rdx18 = add <32 x i32> %bin.rdx16, %rdx.shuf17 569 %13 = extractelement <32 x i32> %bin.rdx18, i32 0 570 ret i32 %13 571} 572 573define i32 @_Z9test_charPcS_i_128(i8* nocapture readonly, i8* nocapture readonly, i32) local_unnamed_addr #0 { 574; SSE2-LABEL: _Z9test_charPcS_i_128: 575; SSE2: # %bb.0: # %entry 576; SSE2-NEXT: movl %edx, %eax 577; SSE2-NEXT: pxor %xmm0, %xmm0 578; SSE2-NEXT: xorl %ecx, %ecx 579; SSE2-NEXT: .p2align 4, 0x90 580; SSE2-NEXT: .LBB4_1: # %vector.body 581; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 582; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero 583; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 584; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero 585; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 586; SSE2-NEXT: psraw $8, %xmm1 587; SSE2-NEXT: psraw $8, %xmm2 588; SSE2-NEXT: pmullw %xmm1, %xmm2 589; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] 590; SSE2-NEXT: psrad $16, %xmm1 591; SSE2-NEXT: paddd %xmm1, %xmm0 592; SSE2-NEXT: addq $16, %rcx 593; SSE2-NEXT: cmpq %rcx, %rax 594; SSE2-NEXT: jne .LBB4_1 595; SSE2-NEXT: # %bb.2: # %middle.block 596; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 597; SSE2-NEXT: paddd %xmm0, %xmm1 598; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3] 599; SSE2-NEXT: paddd %xmm1, %xmm0 600; SSE2-NEXT: movd %xmm0, %eax 601; SSE2-NEXT: retq 602; 603; AVX-LABEL: _Z9test_charPcS_i_128: 604; AVX: # %bb.0: # %entry 605; AVX-NEXT: movl %edx, %eax 606; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0 607; AVX-NEXT: xorl %ecx, %ecx 608; AVX-NEXT: .p2align 4, 0x90 609; AVX-NEXT: .LBB4_1: # %vector.body 610; AVX-NEXT: # =>This Inner Loop Header: Depth=1 611; AVX-NEXT: vpmovsxbd (%rdi,%rcx), %xmm1 612; AVX-NEXT: vpmovsxbd (%rsi,%rcx), %xmm2 613; AVX-NEXT: vpmulld %xmm1, %xmm2, %xmm1 614; AVX-NEXT: vpaddd %xmm0, %xmm1, %xmm0 615; AVX-NEXT: addq $16, %rcx 616; AVX-NEXT: cmpq %rcx, %rax 617; AVX-NEXT: jne .LBB4_1 618; AVX-NEXT: # %bb.2: # %middle.block 619; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 620; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 621; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 622; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 623; AVX-NEXT: vmovd %xmm0, %eax 624; AVX-NEXT: retq 625entry: 626 %3 = zext i32 %2 to i64 627 br label %vector.body 628 629vector.body: 630 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 631 %vec.phi = phi <4 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] 632 %4 = getelementptr inbounds i8, i8* %0, i64 %index 633 %5 = bitcast i8* %4 to <4 x i8>* 634 %wide.load = load <4 x i8>, <4 x i8>* %5, align 1 635 %6 = sext <4 x i8> %wide.load to <4 x i32> 636 %7 = getelementptr inbounds i8, i8* %1, i64 %index 637 %8 = bitcast i8* %7 to <4 x i8>* 638 %wide.load14 = load <4 x i8>, <4 x i8>* %8, align 1 639 %9 = sext <4 x i8> %wide.load14 to <4 x i32> 640 %10 = mul nsw <4 x i32> %9, %6 641 %11 = add nsw <4 x i32> %10, %vec.phi 642 %index.next = add i64 %index, 16 643 %12 = icmp eq i64 %index.next, %3 644 br i1 %12, label %middle.block, label %vector.body 645 646middle.block: 647 %rdx.shuf17 = shufflevector <4 x i32> %11, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> 648 %bin.rdx18 = add <4 x i32> %11, %rdx.shuf17 649 %rdx.shuf19 = shufflevector <4 x i32> %bin.rdx18, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 650 %bin.rdx20 = add <4 x i32> %bin.rdx18, %rdx.shuf19 651 %13 = extractelement <4 x i32> %bin.rdx20, i32 0 652 ret i32 %13 653} 654 655define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly, i32) local_unnamed_addr #0 { 656; SSE2-LABEL: _Z9test_charPcS_i_256: 657; SSE2: # %bb.0: # %entry 658; SSE2-NEXT: movl %edx, %eax 659; SSE2-NEXT: pxor %xmm0, %xmm0 660; SSE2-NEXT: xorl %ecx, %ecx 661; SSE2-NEXT: pxor %xmm1, %xmm1 662; SSE2-NEXT: .p2align 4, 0x90 663; SSE2-NEXT: .LBB5_1: # %vector.body 664; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 665; SSE2-NEXT: movq {{.*#+}} xmm2 = mem[0],zero 666; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 667; SSE2-NEXT: movq {{.*#+}} xmm3 = mem[0],zero 668; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 669; SSE2-NEXT: psraw $8, %xmm2 670; SSE2-NEXT: psraw $8, %xmm3 671; SSE2-NEXT: pmaddwd %xmm2, %xmm3 672; SSE2-NEXT: paddd %xmm3, %xmm1 673; SSE2-NEXT: addq $16, %rcx 674; SSE2-NEXT: cmpq %rcx, %rax 675; SSE2-NEXT: jne .LBB5_1 676; SSE2-NEXT: # %bb.2: # %middle.block 677; SSE2-NEXT: paddd %xmm0, %xmm1 678; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] 679; SSE2-NEXT: paddd %xmm1, %xmm0 680; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 681; SSE2-NEXT: paddd %xmm0, %xmm1 682; SSE2-NEXT: movd %xmm1, %eax 683; SSE2-NEXT: retq 684; 685; AVX1-LABEL: _Z9test_charPcS_i_256: 686; AVX1: # %bb.0: # %entry 687; AVX1-NEXT: movl %edx, %eax 688; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0 689; AVX1-NEXT: xorl %ecx, %ecx 690; AVX1-NEXT: .p2align 4, 0x90 691; AVX1-NEXT: .LBB5_1: # %vector.body 692; AVX1-NEXT: # =>This Inner Loop Header: Depth=1 693; AVX1-NEXT: vpmovsxbw (%rdi,%rcx), %xmm1 694; AVX1-NEXT: vpmovsxbw (%rsi,%rcx), %xmm2 695; AVX1-NEXT: vpmaddwd %xmm1, %xmm2, %xmm1 696; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm1 697; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] 698; AVX1-NEXT: addq $16, %rcx 699; AVX1-NEXT: cmpq %rcx, %rax 700; AVX1-NEXT: jne .LBB5_1 701; AVX1-NEXT: # %bb.2: # %middle.block 702; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 703; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 704; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 705; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 706; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 707; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 708; AVX1-NEXT: vmovd %xmm0, %eax 709; AVX1-NEXT: vzeroupper 710; AVX1-NEXT: retq 711; 712; AVX256-LABEL: _Z9test_charPcS_i_256: 713; AVX256: # %bb.0: # %entry 714; AVX256-NEXT: movl %edx, %eax 715; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0 716; AVX256-NEXT: xorl %ecx, %ecx 717; AVX256-NEXT: .p2align 4, 0x90 718; AVX256-NEXT: .LBB5_1: # %vector.body 719; AVX256-NEXT: # =>This Inner Loop Header: Depth=1 720; AVX256-NEXT: vpmovsxbw (%rdi,%rcx), %xmm1 721; AVX256-NEXT: vpmovsxbw (%rsi,%rcx), %xmm2 722; AVX256-NEXT: vpmaddwd %xmm1, %xmm2, %xmm1 723; AVX256-NEXT: vpaddd %ymm0, %ymm1, %ymm0 724; AVX256-NEXT: addq $16, %rcx 725; AVX256-NEXT: cmpq %rcx, %rax 726; AVX256-NEXT: jne .LBB5_1 727; AVX256-NEXT: # %bb.2: # %middle.block 728; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1 729; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0 730; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 731; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0 732; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 733; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0 734; AVX256-NEXT: vmovd %xmm0, %eax 735; AVX256-NEXT: vzeroupper 736; AVX256-NEXT: retq 737entry: 738 %3 = zext i32 %2 to i64 739 br label %vector.body 740 741vector.body: 742 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 743 %vec.phi = phi <8 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] 744 %4 = getelementptr inbounds i8, i8* %0, i64 %index 745 %5 = bitcast i8* %4 to <8 x i8>* 746 %wide.load = load <8 x i8>, <8 x i8>* %5, align 1 747 %6 = sext <8 x i8> %wide.load to <8 x i32> 748 %7 = getelementptr inbounds i8, i8* %1, i64 %index 749 %8 = bitcast i8* %7 to <8 x i8>* 750 %wide.load14 = load <8 x i8>, <8 x i8>* %8, align 1 751 %9 = sext <8 x i8> %wide.load14 to <8 x i32> 752 %10 = mul nsw <8 x i32> %9, %6 753 %11 = add nsw <8 x i32> %10, %vec.phi 754 %index.next = add i64 %index, 16 755 %12 = icmp eq i64 %index.next, %3 756 br i1 %12, label %middle.block, label %vector.body 757 758middle.block: 759 %rdx.shuf15 = shufflevector <8 x i32> %11, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef> 760 %bin.rdx16 = add <8 x i32> %11, %rdx.shuf15 761 %rdx.shuf17 = shufflevector <8 x i32> %bin.rdx16, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 762 %bin.rdx18 = add <8 x i32> %bin.rdx16, %rdx.shuf17 763 %rdx.shuf19 = shufflevector <8 x i32> %bin.rdx18, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 764 %bin.rdx20 = add <8 x i32> %bin.rdx18, %rdx.shuf19 765 %13 = extractelement <8 x i32> %bin.rdx20, i32 0 766 ret i32 %13 767} 768 769define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly, i32) local_unnamed_addr #0 { 770; SSE2-LABEL: _Z9test_charPcS_i_512: 771; SSE2: # %bb.0: # %entry 772; SSE2-NEXT: movl %edx, %eax 773; SSE2-NEXT: pxor %xmm0, %xmm0 774; SSE2-NEXT: xorl %ecx, %ecx 775; SSE2-NEXT: pxor %xmm2, %xmm2 776; SSE2-NEXT: pxor %xmm1, %xmm1 777; SSE2-NEXT: .p2align 4, 0x90 778; SSE2-NEXT: .LBB6_1: # %vector.body 779; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 780; SSE2-NEXT: movdqu (%rdi,%rcx), %xmm3 781; SSE2-NEXT: movdqu (%rsi,%rcx), %xmm4 782; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15] 783; SSE2-NEXT: psraw $8, %xmm5 784; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15] 785; SSE2-NEXT: psraw $8, %xmm6 786; SSE2-NEXT: pmaddwd %xmm5, %xmm6 787; SSE2-NEXT: paddd %xmm6, %xmm1 788; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 789; SSE2-NEXT: psraw $8, %xmm3 790; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 791; SSE2-NEXT: psraw $8, %xmm4 792; SSE2-NEXT: pmaddwd %xmm3, %xmm4 793; SSE2-NEXT: paddd %xmm4, %xmm2 794; SSE2-NEXT: addq $16, %rcx 795; SSE2-NEXT: cmpq %rcx, %rax 796; SSE2-NEXT: jne .LBB6_1 797; SSE2-NEXT: # %bb.2: # %middle.block 798; SSE2-NEXT: paddd %xmm0, %xmm2 799; SSE2-NEXT: paddd %xmm0, %xmm1 800; SSE2-NEXT: paddd %xmm2, %xmm1 801; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] 802; SSE2-NEXT: paddd %xmm1, %xmm0 803; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 804; SSE2-NEXT: paddd %xmm0, %xmm1 805; SSE2-NEXT: movd %xmm1, %eax 806; SSE2-NEXT: retq 807; 808; AVX1-LABEL: _Z9test_charPcS_i_512: 809; AVX1: # %bb.0: # %entry 810; AVX1-NEXT: movl %edx, %eax 811; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0 812; AVX1-NEXT: xorl %ecx, %ecx 813; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 814; AVX1-NEXT: .p2align 4, 0x90 815; AVX1-NEXT: .LBB6_1: # %vector.body 816; AVX1-NEXT: # =>This Inner Loop Header: Depth=1 817; AVX1-NEXT: vpmovsxbw (%rdi,%rcx), %xmm2 818; AVX1-NEXT: vpmovsxbw 8(%rdi,%rcx), %xmm3 819; AVX1-NEXT: vpmovsxbw (%rsi,%rcx), %xmm4 820; AVX1-NEXT: vpmaddwd %xmm2, %xmm4, %xmm2 821; AVX1-NEXT: vpmovsxbw 8(%rsi,%rcx), %xmm4 822; AVX1-NEXT: vpmaddwd %xmm3, %xmm4, %xmm3 823; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 824; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3 825; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm1 826; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 827; AVX1-NEXT: addq $16, %rcx 828; AVX1-NEXT: cmpq %rcx, %rax 829; AVX1-NEXT: jne .LBB6_1 830; AVX1-NEXT: # %bb.2: # %middle.block 831; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 832; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 833; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2 834; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0 835; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0 836; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 837; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 838; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 839; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 840; AVX1-NEXT: vmovd %xmm0, %eax 841; AVX1-NEXT: vzeroupper 842; AVX1-NEXT: retq 843; 844; AVX2-LABEL: _Z9test_charPcS_i_512: 845; AVX2: # %bb.0: # %entry 846; AVX2-NEXT: movl %edx, %eax 847; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0 848; AVX2-NEXT: xorl %ecx, %ecx 849; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 850; AVX2-NEXT: .p2align 4, 0x90 851; AVX2-NEXT: .LBB6_1: # %vector.body 852; AVX2-NEXT: # =>This Inner Loop Header: Depth=1 853; AVX2-NEXT: vpmovsxbw (%rdi,%rcx), %ymm2 854; AVX2-NEXT: vpmovsxbw (%rsi,%rcx), %ymm3 855; AVX2-NEXT: vpmaddwd %ymm2, %ymm3, %ymm2 856; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1 857; AVX2-NEXT: addq $16, %rcx 858; AVX2-NEXT: cmpq %rcx, %rax 859; AVX2-NEXT: jne .LBB6_1 860; AVX2-NEXT: # %bb.2: # %middle.block 861; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0 862; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 863; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 864; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 865; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 866; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 867; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 868; AVX2-NEXT: vmovd %xmm0, %eax 869; AVX2-NEXT: vzeroupper 870; AVX2-NEXT: retq 871; 872; AVX512-LABEL: _Z9test_charPcS_i_512: 873; AVX512: # %bb.0: # %entry 874; AVX512-NEXT: movl %edx, %eax 875; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0 876; AVX512-NEXT: xorl %ecx, %ecx 877; AVX512-NEXT: .p2align 4, 0x90 878; AVX512-NEXT: .LBB6_1: # %vector.body 879; AVX512-NEXT: # =>This Inner Loop Header: Depth=1 880; AVX512-NEXT: vpmovsxbw (%rdi,%rcx), %ymm1 881; AVX512-NEXT: vpmovsxbw (%rsi,%rcx), %ymm2 882; AVX512-NEXT: vpmaddwd %ymm1, %ymm2, %ymm1 883; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0 884; AVX512-NEXT: addq $16, %rcx 885; AVX512-NEXT: cmpq %rcx, %rax 886; AVX512-NEXT: jne .LBB6_1 887; AVX512-NEXT: # %bb.2: # %middle.block 888; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 889; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0 890; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 891; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 892; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 893; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 894; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 895; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 896; AVX512-NEXT: vmovd %xmm0, %eax 897; AVX512-NEXT: vzeroupper 898; AVX512-NEXT: retq 899entry: 900 %3 = zext i32 %2 to i64 901 br label %vector.body 902 903vector.body: 904 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 905 %vec.phi = phi <16 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] 906 %4 = getelementptr inbounds i8, i8* %0, i64 %index 907 %5 = bitcast i8* %4 to <16 x i8>* 908 %wide.load = load <16 x i8>, <16 x i8>* %5, align 1 909 %6 = sext <16 x i8> %wide.load to <16 x i32> 910 %7 = getelementptr inbounds i8, i8* %1, i64 %index 911 %8 = bitcast i8* %7 to <16 x i8>* 912 %wide.load14 = load <16 x i8>, <16 x i8>* %8, align 1 913 %9 = sext <16 x i8> %wide.load14 to <16 x i32> 914 %10 = mul nsw <16 x i32> %9, %6 915 %11 = add nsw <16 x i32> %10, %vec.phi 916 %index.next = add i64 %index, 16 917 %12 = icmp eq i64 %index.next, %3 918 br i1 %12, label %middle.block, label %vector.body 919 920middle.block: 921 %rdx.shuf = shufflevector <16 x i32> %11, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 922 %bin.rdx = add <16 x i32> %11, %rdx.shuf 923 %rdx.shuf15 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 924 %bin.rdx16 = add <16 x i32> %bin.rdx, %rdx.shuf15 925 %rdx.shuf17 = shufflevector <16 x i32> %bin.rdx16, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 926 %bin.rdx18 = add <16 x i32> %bin.rdx16, %rdx.shuf17 927 %rdx.shuf19 = shufflevector <16 x i32> %bin.rdx18, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 928 %bin.rdx20 = add <16 x i32> %bin.rdx18, %rdx.shuf19 929 %13 = extractelement <16 x i32> %bin.rdx20, i32 0 930 ret i32 %13 931} 932 933define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonly, i32) local_unnamed_addr #0 { 934; SSE2-LABEL: _Z9test_charPcS_i_1024: 935; SSE2: # %bb.0: # %entry 936; SSE2-NEXT: movl %edx, %eax 937; SSE2-NEXT: pxor %xmm8, %xmm8 938; SSE2-NEXT: xorl %ecx, %ecx 939; SSE2-NEXT: pxor %xmm2, %xmm2 940; SSE2-NEXT: pxor %xmm4, %xmm4 941; SSE2-NEXT: pxor %xmm1, %xmm1 942; SSE2-NEXT: pxor %xmm3, %xmm3 943; SSE2-NEXT: .p2align 4, 0x90 944; SSE2-NEXT: .LBB7_1: # %vector.body 945; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 946; SSE2-NEXT: movdqu (%rdi,%rcx), %xmm10 947; SSE2-NEXT: movdqu 16(%rdi,%rcx), %xmm7 948; SSE2-NEXT: movdqu (%rsi,%rcx), %xmm9 949; SSE2-NEXT: movdqu 16(%rsi,%rcx), %xmm0 950; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15] 951; SSE2-NEXT: psraw $8, %xmm5 952; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15] 953; SSE2-NEXT: psraw $8, %xmm6 954; SSE2-NEXT: pmaddwd %xmm5, %xmm6 955; SSE2-NEXT: paddd %xmm6, %xmm3 956; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7] 957; SSE2-NEXT: psraw $8, %xmm5 958; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 959; SSE2-NEXT: psraw $8, %xmm0 960; SSE2-NEXT: pmaddwd %xmm5, %xmm0 961; SSE2-NEXT: paddd %xmm0, %xmm1 962; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm10[8],xmm0[9],xmm10[9],xmm0[10],xmm10[10],xmm0[11],xmm10[11],xmm0[12],xmm10[12],xmm0[13],xmm10[13],xmm0[14],xmm10[14],xmm0[15],xmm10[15] 963; SSE2-NEXT: psraw $8, %xmm0 964; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm9[8],xmm5[9],xmm9[9],xmm5[10],xmm9[10],xmm5[11],xmm9[11],xmm5[12],xmm9[12],xmm5[13],xmm9[13],xmm5[14],xmm9[14],xmm5[15],xmm9[15] 965; SSE2-NEXT: psraw $8, %xmm5 966; SSE2-NEXT: pmaddwd %xmm0, %xmm5 967; SSE2-NEXT: paddd %xmm5, %xmm4 968; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7] 969; SSE2-NEXT: psraw $8, %xmm0 970; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3],xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7] 971; SSE2-NEXT: psraw $8, %xmm5 972; SSE2-NEXT: pmaddwd %xmm0, %xmm5 973; SSE2-NEXT: paddd %xmm5, %xmm2 974; SSE2-NEXT: addq $32, %rcx 975; SSE2-NEXT: cmpq %rcx, %rax 976; SSE2-NEXT: jne .LBB7_1 977; SSE2-NEXT: # %bb.2: # %middle.block 978; SSE2-NEXT: paddd %xmm8, %xmm4 979; SSE2-NEXT: paddd %xmm8, %xmm3 980; SSE2-NEXT: paddd %xmm4, %xmm3 981; SSE2-NEXT: paddd %xmm8, %xmm2 982; SSE2-NEXT: paddd %xmm8, %xmm1 983; SSE2-NEXT: paddd %xmm3, %xmm1 984; SSE2-NEXT: paddd %xmm2, %xmm1 985; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] 986; SSE2-NEXT: paddd %xmm1, %xmm0 987; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 988; SSE2-NEXT: paddd %xmm0, %xmm1 989; SSE2-NEXT: movd %xmm1, %eax 990; SSE2-NEXT: retq 991; 992; AVX1-LABEL: _Z9test_charPcS_i_1024: 993; AVX1: # %bb.0: # %entry 994; AVX1-NEXT: movl %edx, %eax 995; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0 996; AVX1-NEXT: xorl %ecx, %ecx 997; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 998; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 999; AVX1-NEXT: .p2align 4, 0x90 1000; AVX1-NEXT: .LBB7_1: # %vector.body 1001; AVX1-NEXT: # =>This Inner Loop Header: Depth=1 1002; AVX1-NEXT: vpmovsxbw (%rdi,%rcx), %xmm3 1003; AVX1-NEXT: vpmovsxbw 8(%rdi,%rcx), %xmm4 1004; AVX1-NEXT: vpmovsxbw 16(%rdi,%rcx), %xmm5 1005; AVX1-NEXT: vpmovsxbw 24(%rdi,%rcx), %xmm6 1006; AVX1-NEXT: vpmovsxbw (%rsi,%rcx), %xmm7 1007; AVX1-NEXT: vpmaddwd %xmm3, %xmm7, %xmm3 1008; AVX1-NEXT: vpmovsxbw 8(%rsi,%rcx), %xmm7 1009; AVX1-NEXT: vpmaddwd %xmm4, %xmm7, %xmm4 1010; AVX1-NEXT: vpmovsxbw 16(%rsi,%rcx), %xmm7 1011; AVX1-NEXT: vpmaddwd %xmm5, %xmm7, %xmm5 1012; AVX1-NEXT: vpmovsxbw 24(%rsi,%rcx), %xmm7 1013; AVX1-NEXT: vpmaddwd %xmm6, %xmm7, %xmm6 1014; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm7 1015; AVX1-NEXT: vpaddd %xmm7, %xmm6, %xmm6 1016; AVX1-NEXT: vpaddd %xmm2, %xmm5, %xmm2 1017; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm2, %ymm2 1018; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5 1019; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4 1020; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1 1021; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1 1022; AVX1-NEXT: addq $32, %rcx 1023; AVX1-NEXT: cmpq %rcx, %rax 1024; AVX1-NEXT: jne .LBB7_1 1025; AVX1-NEXT: # %bb.2: # %middle.block 1026; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm3 1027; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 1028; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 1029; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2 1030; AVX1-NEXT: vpaddd %xmm5, %xmm2, %xmm2 1031; AVX1-NEXT: vpaddd %xmm2, %xmm5, %xmm2 1032; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm2 1033; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0 1034; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0 1035; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0 1036; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1037; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1038; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1039; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1040; AVX1-NEXT: vmovd %xmm0, %eax 1041; AVX1-NEXT: vzeroupper 1042; AVX1-NEXT: retq 1043; 1044; AVX2-LABEL: _Z9test_charPcS_i_1024: 1045; AVX2: # %bb.0: # %entry 1046; AVX2-NEXT: movl %edx, %eax 1047; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0 1048; AVX2-NEXT: xorl %ecx, %ecx 1049; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 1050; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 1051; AVX2-NEXT: .p2align 4, 0x90 1052; AVX2-NEXT: .LBB7_1: # %vector.body 1053; AVX2-NEXT: # =>This Inner Loop Header: Depth=1 1054; AVX2-NEXT: vpmovsxbw (%rdi,%rcx), %ymm3 1055; AVX2-NEXT: vpmovsxbw 16(%rdi,%rcx), %ymm4 1056; AVX2-NEXT: vpmovsxbw (%rsi,%rcx), %ymm5 1057; AVX2-NEXT: vpmaddwd %ymm3, %ymm5, %ymm3 1058; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1 1059; AVX2-NEXT: vpmovsxbw 16(%rsi,%rcx), %ymm3 1060; AVX2-NEXT: vpmaddwd %ymm4, %ymm3, %ymm3 1061; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2 1062; AVX2-NEXT: addq $32, %rcx 1063; AVX2-NEXT: cmpq %rcx, %rax 1064; AVX2-NEXT: jne .LBB7_1 1065; AVX2-NEXT: # %bb.2: # %middle.block 1066; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm1 1067; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0 1068; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0 1069; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 1070; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1071; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1072; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1073; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1074; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1075; AVX2-NEXT: vmovd %xmm0, %eax 1076; AVX2-NEXT: vzeroupper 1077; AVX2-NEXT: retq 1078; 1079; AVX512F-LABEL: _Z9test_charPcS_i_1024: 1080; AVX512F: # %bb.0: # %entry 1081; AVX512F-NEXT: movl %edx, %eax 1082; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0 1083; AVX512F-NEXT: xorl %ecx, %ecx 1084; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1 1085; AVX512F-NEXT: .p2align 4, 0x90 1086; AVX512F-NEXT: .LBB7_1: # %vector.body 1087; AVX512F-NEXT: # =>This Inner Loop Header: Depth=1 1088; AVX512F-NEXT: vpmovsxbw (%rdi,%rcx), %ymm2 1089; AVX512F-NEXT: vpmovsxbw 16(%rdi,%rcx), %ymm3 1090; AVX512F-NEXT: vpmovsxbw (%rsi,%rcx), %ymm4 1091; AVX512F-NEXT: vpmaddwd %ymm2, %ymm4, %ymm2 1092; AVX512F-NEXT: vpmovsxbw 16(%rsi,%rcx), %ymm4 1093; AVX512F-NEXT: vpmaddwd %ymm3, %ymm4, %ymm3 1094; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 1095; AVX512F-NEXT: vpaddd %zmm1, %zmm2, %zmm1 1096; AVX512F-NEXT: addq $32, %rcx 1097; AVX512F-NEXT: cmpq %rcx, %rax 1098; AVX512F-NEXT: jne .LBB7_1 1099; AVX512F-NEXT: # %bb.2: # %middle.block 1100; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0 1101; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1 1102; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm0 1103; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1 1104; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1105; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1106; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1107; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1108; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1109; AVX512F-NEXT: vmovd %xmm0, %eax 1110; AVX512F-NEXT: vzeroupper 1111; AVX512F-NEXT: retq 1112; 1113; AVX512BW-LABEL: _Z9test_charPcS_i_1024: 1114; AVX512BW: # %bb.0: # %entry 1115; AVX512BW-NEXT: movl %edx, %eax 1116; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0 1117; AVX512BW-NEXT: xorl %ecx, %ecx 1118; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1 1119; AVX512BW-NEXT: .p2align 4, 0x90 1120; AVX512BW-NEXT: .LBB7_1: # %vector.body 1121; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1 1122; AVX512BW-NEXT: vpmovsxbw (%rdi,%rcx), %zmm2 1123; AVX512BW-NEXT: vpmovsxbw (%rsi,%rcx), %zmm3 1124; AVX512BW-NEXT: vpmaddwd %zmm2, %zmm3, %zmm2 1125; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1 1126; AVX512BW-NEXT: addq $32, %rcx 1127; AVX512BW-NEXT: cmpq %rcx, %rax 1128; AVX512BW-NEXT: jne .LBB7_1 1129; AVX512BW-NEXT: # %bb.2: # %middle.block 1130; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0 1131; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1 1132; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0 1133; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1 1134; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1135; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1136; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1137; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1138; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1139; AVX512BW-NEXT: vmovd %xmm0, %eax 1140; AVX512BW-NEXT: vzeroupper 1141; AVX512BW-NEXT: retq 1142entry: 1143 %3 = zext i32 %2 to i64 1144 br label %vector.body 1145 1146vector.body: 1147 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 1148 %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] 1149 %4 = getelementptr inbounds i8, i8* %0, i64 %index 1150 %5 = bitcast i8* %4 to <32 x i8>* 1151 %wide.load = load <32 x i8>, <32 x i8>* %5, align 1 1152 %6 = sext <32 x i8> %wide.load to <32 x i32> 1153 %7 = getelementptr inbounds i8, i8* %1, i64 %index 1154 %8 = bitcast i8* %7 to <32 x i8>* 1155 %wide.load14 = load <32 x i8>, <32 x i8>* %8, align 1 1156 %9 = sext <32 x i8> %wide.load14 to <32 x i32> 1157 %10 = mul nsw <32 x i32> %9, %6 1158 %11 = add nsw <32 x i32> %10, %vec.phi 1159 %index.next = add i64 %index, 32 1160 %12 = icmp eq i64 %index.next, %3 1161 br i1 %12, label %middle.block, label %vector.body 1162 1163middle.block: 1164 %rdx.shuf1 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1165 %bin.rdx1 = add <32 x i32> %11, %rdx.shuf1 1166 %rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1167 %bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf 1168 %rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1169 %bin.rdx32 = add <32 x i32> %bin.rdx, %rdx.shuf15 1170 %rdx.shuf17 = shufflevector <32 x i32> %bin.rdx32, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1171 %bin.rdx18 = add <32 x i32> %bin.rdx32, %rdx.shuf17 1172 %rdx.shuf19 = shufflevector <32 x i32> %bin.rdx18, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1173 %bin.rdx20 = add <32 x i32> %bin.rdx18, %rdx.shuf19 1174 %13 = extractelement <32 x i32> %bin.rdx20, i32 0 1175 ret i32 %13 1176} 1177 1178define i32 @test_unsigned_short_128(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 { 1179; SSE2-LABEL: test_unsigned_short_128: 1180; SSE2: # %bb.0: # %entry 1181; SSE2-NEXT: movl %edx, %eax 1182; SSE2-NEXT: pxor %xmm0, %xmm0 1183; SSE2-NEXT: xorl %ecx, %ecx 1184; SSE2-NEXT: .p2align 4, 0x90 1185; SSE2-NEXT: .LBB8_1: # %vector.body 1186; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 1187; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 1188; SSE2-NEXT: movq {{.*#+}} xmm2 = mem[0],zero 1189; SSE2-NEXT: movdqa %xmm2, %xmm3 1190; SSE2-NEXT: pmulhuw %xmm1, %xmm3 1191; SSE2-NEXT: pmullw %xmm1, %xmm2 1192; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] 1193; SSE2-NEXT: paddd %xmm2, %xmm0 1194; SSE2-NEXT: addq $16, %rcx 1195; SSE2-NEXT: cmpq %rcx, %rax 1196; SSE2-NEXT: jne .LBB8_1 1197; SSE2-NEXT: # %bb.2: # %middle.block 1198; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1199; SSE2-NEXT: paddd %xmm0, %xmm1 1200; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3] 1201; SSE2-NEXT: paddd %xmm1, %xmm0 1202; SSE2-NEXT: movd %xmm0, %eax 1203; SSE2-NEXT: retq 1204; 1205; AVX-LABEL: test_unsigned_short_128: 1206; AVX: # %bb.0: # %entry 1207; AVX-NEXT: movl %edx, %eax 1208; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0 1209; AVX-NEXT: xorl %ecx, %ecx 1210; AVX-NEXT: .p2align 4, 0x90 1211; AVX-NEXT: .LBB8_1: # %vector.body 1212; AVX-NEXT: # =>This Inner Loop Header: Depth=1 1213; AVX-NEXT: vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1214; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1215; AVX-NEXT: vpmulld %xmm1, %xmm2, %xmm1 1216; AVX-NEXT: vpaddd %xmm0, %xmm1, %xmm0 1217; AVX-NEXT: addq $16, %rcx 1218; AVX-NEXT: cmpq %rcx, %rax 1219; AVX-NEXT: jne .LBB8_1 1220; AVX-NEXT: # %bb.2: # %middle.block 1221; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1222; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1223; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1224; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1225; AVX-NEXT: vmovd %xmm0, %eax 1226; AVX-NEXT: retq 1227entry: 1228 %3 = zext i32 %2 to i64 1229 br label %vector.body 1230 1231vector.body: 1232 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 1233 %vec.phi = phi <4 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] 1234 %4 = getelementptr inbounds i16, i16* %0, i64 %index 1235 %5 = bitcast i16* %4 to <4 x i16>* 1236 %wide.load = load <4 x i16>, <4 x i16>* %5, align 2 1237 %6 = zext <4 x i16> %wide.load to <4 x i32> 1238 %7 = getelementptr inbounds i16, i16* %1, i64 %index 1239 %8 = bitcast i16* %7 to <4 x i16>* 1240 %wide.load14 = load <4 x i16>, <4 x i16>* %8, align 2 1241 %9 = zext <4 x i16> %wide.load14 to <4 x i32> 1242 %10 = mul nsw <4 x i32> %9, %6 1243 %11 = add nsw <4 x i32> %10, %vec.phi 1244 %index.next = add i64 %index, 16 1245 %12 = icmp eq i64 %index.next, %3 1246 br i1 %12, label %middle.block, label %vector.body 1247 1248middle.block: 1249 %rdx.shuf15 = shufflevector <4 x i32> %11, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> 1250 %bin.rdx16 = add <4 x i32> %11, %rdx.shuf15 1251 %rdx.shuf17 = shufflevector <4 x i32> %bin.rdx16, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 1252 %bin.rdx18 = add <4 x i32> %bin.rdx16, %rdx.shuf17 1253 %13 = extractelement <4 x i32> %bin.rdx18, i32 0 1254 ret i32 %13 1255} 1256 1257define i32 @test_unsigned_short_256(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 { 1258; SSE2-LABEL: test_unsigned_short_256: 1259; SSE2: # %bb.0: # %entry 1260; SSE2-NEXT: movl %edx, %eax 1261; SSE2-NEXT: pxor %xmm0, %xmm0 1262; SSE2-NEXT: xorl %ecx, %ecx 1263; SSE2-NEXT: pxor %xmm1, %xmm1 1264; SSE2-NEXT: .p2align 4, 0x90 1265; SSE2-NEXT: .LBB9_1: # %vector.body 1266; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 1267; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm2 1268; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm3 1269; SSE2-NEXT: movdqa %xmm3, %xmm4 1270; SSE2-NEXT: pmulhuw %xmm2, %xmm4 1271; SSE2-NEXT: pmullw %xmm2, %xmm3 1272; SSE2-NEXT: movdqa %xmm3, %xmm2 1273; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3] 1274; SSE2-NEXT: paddd %xmm2, %xmm0 1275; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] 1276; SSE2-NEXT: paddd %xmm3, %xmm1 1277; SSE2-NEXT: addq $16, %rcx 1278; SSE2-NEXT: cmpq %rcx, %rax 1279; SSE2-NEXT: jne .LBB9_1 1280; SSE2-NEXT: # %bb.2: # %middle.block 1281; SSE2-NEXT: paddd %xmm1, %xmm0 1282; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1283; SSE2-NEXT: paddd %xmm0, %xmm1 1284; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3] 1285; SSE2-NEXT: paddd %xmm1, %xmm0 1286; SSE2-NEXT: movd %xmm0, %eax 1287; SSE2-NEXT: retq 1288; 1289; AVX1-LABEL: test_unsigned_short_256: 1290; AVX1: # %bb.0: # %entry 1291; AVX1-NEXT: movl %edx, %eax 1292; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0 1293; AVX1-NEXT: xorl %ecx, %ecx 1294; AVX1-NEXT: .p2align 4, 0x90 1295; AVX1-NEXT: .LBB9_1: # %vector.body 1296; AVX1-NEXT: # =>This Inner Loop Header: Depth=1 1297; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1298; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1299; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1300; AVX1-NEXT: vpmulld %xmm1, %xmm3, %xmm1 1301; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1302; AVX1-NEXT: vpmulld %xmm2, %xmm3, %xmm2 1303; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 1304; AVX1-NEXT: vpaddd %xmm3, %xmm1, %xmm1 1305; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0 1306; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 1307; AVX1-NEXT: addq $16, %rcx 1308; AVX1-NEXT: cmpq %rcx, %rax 1309; AVX1-NEXT: jne .LBB9_1 1310; AVX1-NEXT: # %bb.2: # %middle.block 1311; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 1312; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1313; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1314; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1315; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1316; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1317; AVX1-NEXT: vmovd %xmm0, %eax 1318; AVX1-NEXT: vzeroupper 1319; AVX1-NEXT: retq 1320; 1321; AVX256-LABEL: test_unsigned_short_256: 1322; AVX256: # %bb.0: # %entry 1323; AVX256-NEXT: movl %edx, %eax 1324; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0 1325; AVX256-NEXT: xorl %ecx, %ecx 1326; AVX256-NEXT: .p2align 4, 0x90 1327; AVX256-NEXT: .LBB9_1: # %vector.body 1328; AVX256-NEXT: # =>This Inner Loop Header: Depth=1 1329; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1330; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1331; AVX256-NEXT: vpmulld %ymm1, %ymm2, %ymm1 1332; AVX256-NEXT: vpaddd %ymm0, %ymm1, %ymm0 1333; AVX256-NEXT: addq $16, %rcx 1334; AVX256-NEXT: cmpq %rcx, %rax 1335; AVX256-NEXT: jne .LBB9_1 1336; AVX256-NEXT: # %bb.2: # %middle.block 1337; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1 1338; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1339; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1340; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1341; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1342; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1343; AVX256-NEXT: vmovd %xmm0, %eax 1344; AVX256-NEXT: vzeroupper 1345; AVX256-NEXT: retq 1346entry: 1347 %3 = zext i32 %2 to i64 1348 br label %vector.body 1349 1350vector.body: 1351 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 1352 %vec.phi = phi <8 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] 1353 %4 = getelementptr inbounds i16, i16* %0, i64 %index 1354 %5 = bitcast i16* %4 to <8 x i16>* 1355 %wide.load = load <8 x i16>, <8 x i16>* %5, align 2 1356 %6 = zext <8 x i16> %wide.load to <8 x i32> 1357 %7 = getelementptr inbounds i16, i16* %1, i64 %index 1358 %8 = bitcast i16* %7 to <8 x i16>* 1359 %wide.load14 = load <8 x i16>, <8 x i16>* %8, align 2 1360 %9 = zext <8 x i16> %wide.load14 to <8 x i32> 1361 %10 = mul nsw <8 x i32> %9, %6 1362 %11 = add nsw <8 x i32> %10, %vec.phi 1363 %index.next = add i64 %index, 16 1364 %12 = icmp eq i64 %index.next, %3 1365 br i1 %12, label %middle.block, label %vector.body 1366 1367middle.block: 1368 %rdx.shuf = shufflevector <8 x i32> %11, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef> 1369 %bin.rdx = add <8 x i32> %11, %rdx.shuf 1370 %rdx.shuf15 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1371 %bin.rdx16 = add <8 x i32> %bin.rdx, %rdx.shuf15 1372 %rdx.shuf17 = shufflevector <8 x i32> %bin.rdx16, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1373 %bin.rdx18 = add <8 x i32> %bin.rdx16, %rdx.shuf17 1374 %13 = extractelement <8 x i32> %bin.rdx18, i32 0 1375 ret i32 %13 1376} 1377 1378define i32 @test_unsigned_short_512(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 { 1379; SSE2-LABEL: test_unsigned_short_512: 1380; SSE2: # %bb.0: # %entry 1381; SSE2-NEXT: movl %edx, %eax 1382; SSE2-NEXT: pxor %xmm0, %xmm0 1383; SSE2-NEXT: xorl %ecx, %ecx 1384; SSE2-NEXT: pxor %xmm1, %xmm1 1385; SSE2-NEXT: pxor %xmm3, %xmm3 1386; SSE2-NEXT: pxor %xmm2, %xmm2 1387; SSE2-NEXT: .p2align 4, 0x90 1388; SSE2-NEXT: .LBB10_1: # %vector.body 1389; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 1390; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm4 1391; SSE2-NEXT: movdqu 16(%rdi,%rcx,2), %xmm8 1392; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm6 1393; SSE2-NEXT: movdqu 16(%rsi,%rcx,2), %xmm7 1394; SSE2-NEXT: movdqa %xmm6, %xmm5 1395; SSE2-NEXT: pmulhuw %xmm4, %xmm5 1396; SSE2-NEXT: pmullw %xmm4, %xmm6 1397; SSE2-NEXT: movdqa %xmm6, %xmm4 1398; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3] 1399; SSE2-NEXT: paddd %xmm4, %xmm0 1400; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] 1401; SSE2-NEXT: paddd %xmm6, %xmm1 1402; SSE2-NEXT: movdqa %xmm7, %xmm4 1403; SSE2-NEXT: pmulhuw %xmm8, %xmm4 1404; SSE2-NEXT: pmullw %xmm8, %xmm7 1405; SSE2-NEXT: movdqa %xmm7, %xmm5 1406; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3] 1407; SSE2-NEXT: paddd %xmm5, %xmm3 1408; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7] 1409; SSE2-NEXT: paddd %xmm7, %xmm2 1410; SSE2-NEXT: addq $16, %rcx 1411; SSE2-NEXT: cmpq %rcx, %rax 1412; SSE2-NEXT: jne .LBB10_1 1413; SSE2-NEXT: # %bb.2: # %middle.block 1414; SSE2-NEXT: paddd %xmm3, %xmm0 1415; SSE2-NEXT: paddd %xmm2, %xmm1 1416; SSE2-NEXT: paddd %xmm0, %xmm1 1417; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] 1418; SSE2-NEXT: paddd %xmm1, %xmm0 1419; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1420; SSE2-NEXT: paddd %xmm0, %xmm1 1421; SSE2-NEXT: movd %xmm1, %eax 1422; SSE2-NEXT: retq 1423; 1424; AVX1-LABEL: test_unsigned_short_512: 1425; AVX1: # %bb.0: # %entry 1426; AVX1-NEXT: movl %edx, %eax 1427; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0 1428; AVX1-NEXT: xorl %ecx, %ecx 1429; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 1430; AVX1-NEXT: .p2align 4, 0x90 1431; AVX1-NEXT: .LBB10_1: # %vector.body 1432; AVX1-NEXT: # =>This Inner Loop Header: Depth=1 1433; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1434; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1435; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1436; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1437; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1438; AVX1-NEXT: vpmulld %xmm2, %xmm6, %xmm2 1439; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1440; AVX1-NEXT: vpmulld %xmm3, %xmm6, %xmm3 1441; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1442; AVX1-NEXT: vpmulld %xmm4, %xmm6, %xmm4 1443; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1444; AVX1-NEXT: vpmulld %xmm5, %xmm6, %xmm5 1445; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm6 1446; AVX1-NEXT: vpaddd %xmm6, %xmm2, %xmm2 1447; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1 1448; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 1449; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 1450; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm2 1451; AVX1-NEXT: vpaddd %xmm0, %xmm5, %xmm0 1452; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 1453; AVX1-NEXT: addq $16, %rcx 1454; AVX1-NEXT: cmpq %rcx, %rax 1455; AVX1-NEXT: jne .LBB10_1 1456; AVX1-NEXT: # %bb.2: # %middle.block 1457; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 1458; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 1459; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2 1460; AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm1 1461; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1462; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1463; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1464; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1465; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1466; AVX1-NEXT: vmovd %xmm0, %eax 1467; AVX1-NEXT: vzeroupper 1468; AVX1-NEXT: retq 1469; 1470; AVX2-LABEL: test_unsigned_short_512: 1471; AVX2: # %bb.0: # %entry 1472; AVX2-NEXT: movl %edx, %eax 1473; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0 1474; AVX2-NEXT: xorl %ecx, %ecx 1475; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 1476; AVX2-NEXT: .p2align 4, 0x90 1477; AVX2-NEXT: .LBB10_1: # %vector.body 1478; AVX2-NEXT: # =>This Inner Loop Header: Depth=1 1479; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1480; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1481; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1482; AVX2-NEXT: vpmulld %ymm2, %ymm4, %ymm2 1483; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1 1484; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1485; AVX2-NEXT: vpmulld %ymm3, %ymm2, %ymm2 1486; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0 1487; AVX2-NEXT: addq $16, %rcx 1488; AVX2-NEXT: cmpq %rcx, %rax 1489; AVX2-NEXT: jne .LBB10_1 1490; AVX2-NEXT: # %bb.2: # %middle.block 1491; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0 1492; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 1493; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1494; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1495; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1496; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1497; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1498; AVX2-NEXT: vmovd %xmm0, %eax 1499; AVX2-NEXT: vzeroupper 1500; AVX2-NEXT: retq 1501; 1502; AVX512-LABEL: test_unsigned_short_512: 1503; AVX512: # %bb.0: # %entry 1504; AVX512-NEXT: movl %edx, %eax 1505; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0 1506; AVX512-NEXT: xorl %ecx, %ecx 1507; AVX512-NEXT: .p2align 4, 0x90 1508; AVX512-NEXT: .LBB10_1: # %vector.body 1509; AVX512-NEXT: # =>This Inner Loop Header: Depth=1 1510; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 1511; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 1512; AVX512-NEXT: vpmulld %zmm1, %zmm2, %zmm1 1513; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0 1514; AVX512-NEXT: addq $16, %rcx 1515; AVX512-NEXT: cmpq %rcx, %rax 1516; AVX512-NEXT: jne .LBB10_1 1517; AVX512-NEXT: # %bb.2: # %middle.block 1518; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 1519; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0 1520; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 1521; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1522; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1523; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1524; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1525; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1526; AVX512-NEXT: vmovd %xmm0, %eax 1527; AVX512-NEXT: vzeroupper 1528; AVX512-NEXT: retq 1529entry: 1530 %3 = zext i32 %2 to i64 1531 br label %vector.body 1532 1533vector.body: 1534 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 1535 %vec.phi = phi <16 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] 1536 %4 = getelementptr inbounds i16, i16* %0, i64 %index 1537 %5 = bitcast i16* %4 to <16 x i16>* 1538 %wide.load = load <16 x i16>, <16 x i16>* %5, align 2 1539 %6 = zext <16 x i16> %wide.load to <16 x i32> 1540 %7 = getelementptr inbounds i16, i16* %1, i64 %index 1541 %8 = bitcast i16* %7 to <16 x i16>* 1542 %wide.load14 = load <16 x i16>, <16 x i16>* %8, align 2 1543 %9 = zext <16 x i16> %wide.load14 to <16 x i32> 1544 %10 = mul nsw <16 x i32> %9, %6 1545 %11 = add nsw <16 x i32> %10, %vec.phi 1546 %index.next = add i64 %index, 16 1547 %12 = icmp eq i64 %index.next, %3 1548 br i1 %12, label %middle.block, label %vector.body 1549 1550middle.block: 1551 %rdx.shuf1 = shufflevector <16 x i32> %11, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1552 %bin.rdx1 = add <16 x i32> %11, %rdx.shuf1 1553 %rdx.shuf = shufflevector <16 x i32> %bin.rdx1, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1554 %bin.rdx = add <16 x i32> %bin.rdx1, %rdx.shuf 1555 %rdx.shuf15 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1556 %bin.rdx16 = add <16 x i32> %bin.rdx, %rdx.shuf15 1557 %rdx.shuf17 = shufflevector <16 x i32> %bin.rdx16, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1558 %bin.rdx18 = add <16 x i32> %bin.rdx16, %rdx.shuf17 1559 %13 = extractelement <16 x i32> %bin.rdx18, i32 0 1560 ret i32 %13 1561} 1562 1563define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 { 1564; SSE2-LABEL: test_unsigned_short_1024: 1565; SSE2: # %bb.0: # %entry 1566; SSE2-NEXT: movl %edx, %eax 1567; SSE2-NEXT: pxor %xmm8, %xmm8 1568; SSE2-NEXT: xorl %ecx, %ecx 1569; SSE2-NEXT: pxor %xmm3, %xmm3 1570; SSE2-NEXT: pxor %xmm9, %xmm9 1571; SSE2-NEXT: pxor %xmm10, %xmm10 1572; SSE2-NEXT: pxor %xmm4, %xmm4 1573; SSE2-NEXT: pxor %xmm6, %xmm6 1574; SSE2-NEXT: pxor %xmm5, %xmm5 1575; SSE2-NEXT: pxor %xmm7, %xmm7 1576; SSE2-NEXT: .p2align 4, 0x90 1577; SSE2-NEXT: .LBB11_1: # %vector.body 1578; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 1579; SSE2-NEXT: movdqu 48(%rdi,%rcx,2), %xmm0 1580; SSE2-NEXT: movdqu 48(%rsi,%rcx,2), %xmm1 1581; SSE2-NEXT: movdqa %xmm1, %xmm2 1582; SSE2-NEXT: pmulhuw %xmm0, %xmm2 1583; SSE2-NEXT: pmullw %xmm0, %xmm1 1584; SSE2-NEXT: movdqa %xmm1, %xmm0 1585; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] 1586; SSE2-NEXT: paddd %xmm0, %xmm7 1587; SSE2-NEXT: movdqu 32(%rdi,%rcx,2), %xmm0 1588; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] 1589; SSE2-NEXT: movdqu 32(%rsi,%rcx,2), %xmm2 1590; SSE2-NEXT: paddd %xmm1, %xmm5 1591; SSE2-NEXT: movdqa %xmm2, %xmm1 1592; SSE2-NEXT: pmulhuw %xmm0, %xmm1 1593; SSE2-NEXT: pmullw %xmm0, %xmm2 1594; SSE2-NEXT: movdqa %xmm2, %xmm0 1595; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] 1596; SSE2-NEXT: paddd %xmm0, %xmm6 1597; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm0 1598; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] 1599; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm1 1600; SSE2-NEXT: paddd %xmm2, %xmm4 1601; SSE2-NEXT: movdqa %xmm1, %xmm2 1602; SSE2-NEXT: pmulhuw %xmm0, %xmm2 1603; SSE2-NEXT: pmullw %xmm0, %xmm1 1604; SSE2-NEXT: movdqa %xmm1, %xmm0 1605; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] 1606; SSE2-NEXT: paddd %xmm0, %xmm8 1607; SSE2-NEXT: movdqu 16(%rdi,%rcx,2), %xmm0 1608; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] 1609; SSE2-NEXT: movdqu 16(%rsi,%rcx,2), %xmm2 1610; SSE2-NEXT: paddd %xmm1, %xmm3 1611; SSE2-NEXT: movdqa %xmm2, %xmm1 1612; SSE2-NEXT: pmulhuw %xmm0, %xmm1 1613; SSE2-NEXT: pmullw %xmm0, %xmm2 1614; SSE2-NEXT: movdqa %xmm2, %xmm0 1615; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] 1616; SSE2-NEXT: paddd %xmm0, %xmm9 1617; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] 1618; SSE2-NEXT: paddd %xmm2, %xmm10 1619; SSE2-NEXT: addq $16, %rcx 1620; SSE2-NEXT: cmpq %rcx, %rax 1621; SSE2-NEXT: jne .LBB11_1 1622; SSE2-NEXT: # %bb.2: # %middle.block 1623; SSE2-NEXT: paddd %xmm6, %xmm3 1624; SSE2-NEXT: paddd %xmm7, %xmm10 1625; SSE2-NEXT: paddd %xmm3, %xmm10 1626; SSE2-NEXT: paddd %xmm4, %xmm8 1627; SSE2-NEXT: paddd %xmm5, %xmm9 1628; SSE2-NEXT: paddd %xmm10, %xmm9 1629; SSE2-NEXT: paddd %xmm8, %xmm9 1630; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm9[2,3,0,1] 1631; SSE2-NEXT: paddd %xmm9, %xmm0 1632; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1633; SSE2-NEXT: paddd %xmm0, %xmm1 1634; SSE2-NEXT: movd %xmm1, %eax 1635; SSE2-NEXT: retq 1636; 1637; AVX1-LABEL: test_unsigned_short_1024: 1638; AVX1: # %bb.0: # %entry 1639; AVX1-NEXT: movl %edx, %eax 1640; AVX1-NEXT: vpxor %xmm8, %xmm8, %xmm8 1641; AVX1-NEXT: xorl %ecx, %ecx 1642; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 1643; AVX1-NEXT: vpxor %xmm9, %xmm9, %xmm9 1644; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 1645; AVX1-NEXT: .p2align 4, 0x90 1646; AVX1-NEXT: .LBB11_1: # %vector.body 1647; AVX1-NEXT: # =>This Inner Loop Header: Depth=1 1648; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1649; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1650; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1651; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm7 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1652; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1653; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm12 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1654; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm10 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1655; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm11 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1656; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1657; AVX1-NEXT: vpmulld %xmm4, %xmm1, %xmm1 1658; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1659; AVX1-NEXT: vpmulld %xmm5, %xmm4, %xmm4 1660; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1661; AVX1-NEXT: vpmulld %xmm6, %xmm5, %xmm5 1662; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1663; AVX1-NEXT: vpmulld %xmm7, %xmm6, %xmm6 1664; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm7 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1665; AVX1-NEXT: vpmulld %xmm0, %xmm7, %xmm13 1666; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm7 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1667; AVX1-NEXT: vpmulld %xmm12, %xmm7, %xmm7 1668; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1669; AVX1-NEXT: vpmulld %xmm10, %xmm0, %xmm10 1670; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero 1671; AVX1-NEXT: vpmulld %xmm11, %xmm0, %xmm11 1672; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm0 1673; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0 1674; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm1 1675; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm2 1676; AVX1-NEXT: vextractf128 $1, %ymm8, %xmm0 1677; AVX1-NEXT: vpaddd %xmm0, %xmm5, %xmm0 1678; AVX1-NEXT: vpaddd %xmm6, %xmm8, %xmm1 1679; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm8 1680; AVX1-NEXT: vextractf128 $1, %ymm9, %xmm0 1681; AVX1-NEXT: vpaddd %xmm0, %xmm13, %xmm0 1682; AVX1-NEXT: vpaddd %xmm7, %xmm9, %xmm1 1683; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm9 1684; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm0 1685; AVX1-NEXT: vpaddd %xmm0, %xmm10, %xmm0 1686; AVX1-NEXT: vpaddd %xmm3, %xmm11, %xmm1 1687; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm3 1688; AVX1-NEXT: addq $16, %rcx 1689; AVX1-NEXT: cmpq %rcx, %rax 1690; AVX1-NEXT: jne .LBB11_1 1691; AVX1-NEXT: # %bb.2: # %middle.block 1692; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm0 1693; AVX1-NEXT: vextractf128 $1, %ymm8, %xmm1 1694; AVX1-NEXT: vextractf128 $1, %ymm9, %xmm4 1695; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2 1696; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3 1697; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2 1698; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm2 1699; AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm1 1700; AVX1-NEXT: vpaddd %xmm0, %xmm9, %xmm0 1701; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1702; AVX1-NEXT: vpaddd %xmm0, %xmm8, %xmm0 1703; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1704; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1705; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1706; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1707; AVX1-NEXT: vmovd %xmm0, %eax 1708; AVX1-NEXT: vzeroupper 1709; AVX1-NEXT: retq 1710; 1711; AVX2-LABEL: test_unsigned_short_1024: 1712; AVX2: # %bb.0: # %entry 1713; AVX2-NEXT: movl %edx, %eax 1714; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0 1715; AVX2-NEXT: xorl %ecx, %ecx 1716; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 1717; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 1718; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3 1719; AVX2-NEXT: .p2align 4, 0x90 1720; AVX2-NEXT: .LBB11_1: # %vector.body 1721; AVX2-NEXT: # =>This Inner Loop Header: Depth=1 1722; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1723; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1724; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1725; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm7 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1726; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm8 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1727; AVX2-NEXT: vpmulld %ymm4, %ymm8, %ymm4 1728; AVX2-NEXT: vpaddd %ymm2, %ymm4, %ymm2 1729; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1730; AVX2-NEXT: vpmulld %ymm5, %ymm4, %ymm4 1731; AVX2-NEXT: vpaddd %ymm1, %ymm4, %ymm1 1732; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1733; AVX2-NEXT: vpmulld %ymm6, %ymm4, %ymm4 1734; AVX2-NEXT: vpaddd %ymm0, %ymm4, %ymm0 1735; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1736; AVX2-NEXT: vpmulld %ymm7, %ymm4, %ymm4 1737; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3 1738; AVX2-NEXT: addq $16, %rcx 1739; AVX2-NEXT: cmpq %rcx, %rax 1740; AVX2-NEXT: jne .LBB11_1 1741; AVX2-NEXT: # %bb.2: # %middle.block 1742; AVX2-NEXT: vpaddd %ymm2, %ymm0, %ymm0 1743; AVX2-NEXT: vpaddd %ymm3, %ymm1, %ymm1 1744; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0 1745; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 1746; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1747; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1748; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1749; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1750; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1751; AVX2-NEXT: vmovd %xmm0, %eax 1752; AVX2-NEXT: vzeroupper 1753; AVX2-NEXT: retq 1754; 1755; AVX512-LABEL: test_unsigned_short_1024: 1756; AVX512: # %bb.0: # %entry 1757; AVX512-NEXT: movl %edx, %eax 1758; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0 1759; AVX512-NEXT: xorl %ecx, %ecx 1760; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1 1761; AVX512-NEXT: .p2align 4, 0x90 1762; AVX512-NEXT: .LBB11_1: # %vector.body 1763; AVX512-NEXT: # =>This Inner Loop Header: Depth=1 1764; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 1765; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 1766; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 1767; AVX512-NEXT: vpmulld %zmm2, %zmm4, %zmm2 1768; AVX512-NEXT: vpaddd %zmm1, %zmm2, %zmm1 1769; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 1770; AVX512-NEXT: vpmulld %zmm3, %zmm2, %zmm2 1771; AVX512-NEXT: vpaddd %zmm0, %zmm2, %zmm0 1772; AVX512-NEXT: addq $16, %rcx 1773; AVX512-NEXT: cmpq %rcx, %rax 1774; AVX512-NEXT: jne .LBB11_1 1775; AVX512-NEXT: # %bb.2: # %middle.block 1776; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0 1777; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 1778; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0 1779; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 1780; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1781; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 1782; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1783; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 1784; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 1785; AVX512-NEXT: vmovd %xmm0, %eax 1786; AVX512-NEXT: vzeroupper 1787; AVX512-NEXT: retq 1788entry: 1789 %3 = zext i32 %2 to i64 1790 br label %vector.body 1791 1792vector.body: 1793 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 1794 %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] 1795 %4 = getelementptr inbounds i16, i16* %0, i64 %index 1796 %5 = bitcast i16* %4 to <32 x i16>* 1797 %wide.load = load <32 x i16>, <32 x i16>* %5, align 2 1798 %6 = zext <32 x i16> %wide.load to <32 x i32> 1799 %7 = getelementptr inbounds i16, i16* %1, i64 %index 1800 %8 = bitcast i16* %7 to <32 x i16>* 1801 %wide.load14 = load <32 x i16>, <32 x i16>* %8, align 2 1802 %9 = zext <32 x i16> %wide.load14 to <32 x i32> 1803 %10 = mul nsw <32 x i32> %9, %6 1804 %11 = add nsw <32 x i32> %10, %vec.phi 1805 %index.next = add i64 %index, 16 1806 %12 = icmp eq i64 %index.next, %3 1807 br i1 %12, label %middle.block, label %vector.body 1808 1809middle.block: 1810 %rdx.shuf2 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1811 %bin.rdx2 = add <32 x i32> %11, %rdx.shuf2 1812 %rdx.shuf1 = shufflevector <32 x i32> %bin.rdx2, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1813 %bin.rdx1 = add <32 x i32> %bin.rdx2, %rdx.shuf1 1814 %rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1815 %bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf 1816 %rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1817 %bin.rdx16 = add <32 x i32> %bin.rdx, %rdx.shuf15 1818 %rdx.shuf17 = shufflevector <32 x i32> %bin.rdx16, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 1819 %bin.rdx18 = add <32 x i32> %bin.rdx16, %rdx.shuf17 1820 %13 = extractelement <32 x i32> %bin.rdx18, i32 0 1821 ret i32 %13 1822} 1823 1824define <4 x i32> @pmaddwd_8(<8 x i16> %A, <8 x i16> %B) { 1825; SSE2-LABEL: pmaddwd_8: 1826; SSE2: # %bb.0: 1827; SSE2-NEXT: pmaddwd %xmm1, %xmm0 1828; SSE2-NEXT: retq 1829; 1830; AVX-LABEL: pmaddwd_8: 1831; AVX: # %bb.0: 1832; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 1833; AVX-NEXT: retq 1834 %a = sext <8 x i16> %A to <8 x i32> 1835 %b = sext <8 x i16> %B to <8 x i32> 1836 %m = mul nsw <8 x i32> %a, %b 1837 %odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 1838 %even = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 1839 %ret = add <4 x i32> %odd, %even 1840 ret <4 x i32> %ret 1841} 1842 1843define <4 x i32> @pmaddwd_8_swapped(<8 x i16> %A, <8 x i16> %B) { 1844; SSE2-LABEL: pmaddwd_8_swapped: 1845; SSE2: # %bb.0: 1846; SSE2-NEXT: pmaddwd %xmm1, %xmm0 1847; SSE2-NEXT: retq 1848; 1849; AVX-LABEL: pmaddwd_8_swapped: 1850; AVX: # %bb.0: 1851; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 1852; AVX-NEXT: retq 1853 %a = sext <8 x i16> %A to <8 x i32> 1854 %b = sext <8 x i16> %B to <8 x i32> 1855 %m = mul nsw <8 x i32> %a, %b 1856 %odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 1857 %even = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 1858 %ret = add <4 x i32> %even, %odd 1859 ret <4 x i32> %ret 1860} 1861 1862define <4 x i32> @larger_mul(<16 x i16> %A, <16 x i16> %B) { 1863; SSE2-LABEL: larger_mul: 1864; SSE2: # %bb.0: 1865; SSE2-NEXT: movdqa %xmm0, %xmm1 1866; SSE2-NEXT: pmulhw %xmm2, %xmm1 1867; SSE2-NEXT: pmullw %xmm2, %xmm0 1868; SSE2-NEXT: movdqa %xmm0, %xmm2 1869; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] 1870; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] 1871; SSE2-NEXT: movdqa %xmm0, %xmm1 1872; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2] 1873; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3] 1874; SSE2-NEXT: paddd %xmm1, %xmm0 1875; SSE2-NEXT: retq 1876; 1877; AVX1-LABEL: larger_mul: 1878; AVX1: # %bb.0: 1879; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 1880; AVX1-NEXT: vzeroupper 1881; AVX1-NEXT: retq 1882; 1883; AVX2-LABEL: larger_mul: 1884; AVX2: # %bb.0: 1885; AVX2-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 1886; AVX2-NEXT: vzeroupper 1887; AVX2-NEXT: retq 1888; 1889; AVX512-LABEL: larger_mul: 1890; AVX512: # %bb.0: 1891; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0 1892; AVX512-NEXT: vpmovsxwd %ymm1, %zmm1 1893; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0 1894; AVX512-NEXT: vpextrd $2, %xmm0, %eax 1895; AVX512-NEXT: vpinsrd $1, %eax, %xmm0, %xmm1 1896; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2 1897; AVX512-NEXT: vmovd %xmm2, %eax 1898; AVX512-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1 1899; AVX512-NEXT: vpextrd $2, %xmm2, %eax 1900; AVX512-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1 1901; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3] 1902; AVX512-NEXT: vpextrd $1, %xmm2, %eax 1903; AVX512-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0 1904; AVX512-NEXT: vpextrd $3, %xmm2, %eax 1905; AVX512-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0 1906; AVX512-NEXT: vpaddd %xmm0, %xmm1, %xmm0 1907; AVX512-NEXT: vzeroupper 1908; AVX512-NEXT: retq 1909 %a = sext <16 x i16> %A to <16 x i32> 1910 %b = sext <16 x i16> %B to <16 x i32> 1911 %m = mul nsw <16 x i32> %a, %b 1912 %odd = shufflevector <16 x i32> %m, <16 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 1913 %even = shufflevector <16 x i32> %m, <16 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 1914 %ret = add <4 x i32> %odd, %even 1915 ret <4 x i32> %ret 1916} 1917 1918define <8 x i32> @pmaddwd_16(<16 x i16> %A, <16 x i16> %B) { 1919; SSE2-LABEL: pmaddwd_16: 1920; SSE2: # %bb.0: 1921; SSE2-NEXT: pmaddwd %xmm2, %xmm0 1922; SSE2-NEXT: pmaddwd %xmm3, %xmm1 1923; SSE2-NEXT: retq 1924; 1925; AVX1-LABEL: pmaddwd_16: 1926; AVX1: # %bb.0: 1927; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 1928; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 1929; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2 1930; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 1931; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 1932; AVX1-NEXT: retq 1933; 1934; AVX256-LABEL: pmaddwd_16: 1935; AVX256: # %bb.0: 1936; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0 1937; AVX256-NEXT: retq 1938 %a = sext <16 x i16> %A to <16 x i32> 1939 %b = sext <16 x i16> %B to <16 x i32> 1940 %m = mul nsw <16 x i32> %a, %b 1941 %odd = shufflevector <16 x i32> %m, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14> 1942 %even = shufflevector <16 x i32> %m, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15> 1943 %ret = add <8 x i32> %odd, %even 1944 ret <8 x i32> %ret 1945} 1946 1947define <16 x i32> @pmaddwd_32(<32 x i16> %A, <32 x i16> %B) { 1948; SSE2-LABEL: pmaddwd_32: 1949; SSE2: # %bb.0: 1950; SSE2-NEXT: pmaddwd %xmm4, %xmm0 1951; SSE2-NEXT: pmaddwd %xmm5, %xmm1 1952; SSE2-NEXT: pmaddwd %xmm6, %xmm2 1953; SSE2-NEXT: pmaddwd %xmm7, %xmm3 1954; SSE2-NEXT: retq 1955; 1956; AVX1-LABEL: pmaddwd_32: 1957; AVX1: # %bb.0: 1958; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 1959; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 1960; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm6 1961; AVX1-NEXT: vpmaddwd %xmm6, %xmm4, %xmm4 1962; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6 1963; AVX1-NEXT: vpmaddwd %xmm6, %xmm5, %xmm5 1964; AVX1-NEXT: vpmaddwd %xmm2, %xmm0, %xmm0 1965; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm0 1966; AVX1-NEXT: vpmaddwd %xmm3, %xmm1, %xmm1 1967; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1 1968; AVX1-NEXT: retq 1969; 1970; AVX2-LABEL: pmaddwd_32: 1971; AVX2: # %bb.0: 1972; AVX2-NEXT: vpmaddwd %ymm2, %ymm0, %ymm0 1973; AVX2-NEXT: vpmaddwd %ymm3, %ymm1, %ymm1 1974; AVX2-NEXT: retq 1975; 1976; AVX512F-LABEL: pmaddwd_32: 1977; AVX512F: # %bb.0: 1978; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2 1979; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3 1980; AVX512F-NEXT: vpmaddwd %ymm2, %ymm3, %ymm2 1981; AVX512F-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0 1982; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 1983; AVX512F-NEXT: retq 1984; 1985; AVX512BW-LABEL: pmaddwd_32: 1986; AVX512BW: # %bb.0: 1987; AVX512BW-NEXT: vpmaddwd %zmm1, %zmm0, %zmm0 1988; AVX512BW-NEXT: retq 1989 %a = sext <32 x i16> %A to <32 x i32> 1990 %b = sext <32 x i16> %B to <32 x i32> 1991 %m = mul nsw <32 x i32> %a, %b 1992 %odd = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30> 1993 %even = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31> 1994 %ret = add <16 x i32> %odd, %even 1995 ret <16 x i32> %ret 1996} 1997 1998define <4 x i32> @pmaddwd_const(<8 x i16> %A) { 1999; SSE2-LABEL: pmaddwd_const: 2000; SSE2: # %bb.0: 2001; SSE2-NEXT: pmaddwd {{.*}}(%rip), %xmm0 2002; SSE2-NEXT: retq 2003; 2004; AVX-LABEL: pmaddwd_const: 2005; AVX: # %bb.0: 2006; AVX-NEXT: vpmaddwd {{.*}}(%rip), %xmm0, %xmm0 2007; AVX-NEXT: retq 2008 %a = sext <8 x i16> %A to <8 x i32> 2009 %m = mul nsw <8 x i32> %a, <i32 32767, i32 -32768, i32 0, i32 0, i32 1, i32 7, i32 42, i32 32> 2010 %odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 2011 %even = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 2012 %ret = add <4 x i32> %odd, %even 2013 ret <4 x i32> %ret 2014} 2015 2016; Do not select unsigned i16 multiplication 2017define <4 x i32> @pmaddwd_negative1(<8 x i16> %A, <8 x i16> %B) { 2018; SSE2-LABEL: pmaddwd_negative1: 2019; SSE2: # %bb.0: 2020; SSE2-NEXT: movdqa %xmm0, %xmm2 2021; SSE2-NEXT: pmulhuw %xmm1, %xmm2 2022; SSE2-NEXT: pmullw %xmm1, %xmm0 2023; SSE2-NEXT: movdqa %xmm0, %xmm1 2024; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] 2025; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] 2026; SSE2-NEXT: movdqa %xmm0, %xmm2 2027; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm1[0,2] 2028; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3] 2029; SSE2-NEXT: paddd %xmm2, %xmm0 2030; SSE2-NEXT: retq 2031; 2032; AVX1-LABEL: pmaddwd_negative1: 2033; AVX1: # %bb.0: 2034; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 2035; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] 2036; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 2037; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] 2038; AVX1-NEXT: vpmulld %xmm2, %xmm3, %xmm2 2039; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 2040; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0 2041; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0 2042; AVX1-NEXT: retq 2043; 2044; AVX256-LABEL: pmaddwd_negative1: 2045; AVX256: # %bb.0: 2046; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 2047; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 2048; AVX256-NEXT: vpmulld %ymm1, %ymm0, %ymm0 2049; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1 2050; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0 2051; AVX256-NEXT: vzeroupper 2052; AVX256-NEXT: retq 2053 %a = zext <8 x i16> %A to <8 x i32> 2054 %b = zext <8 x i16> %B to <8 x i32> 2055 %m = mul nuw <8 x i32> %a, %b 2056 %odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 2057 %even = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 2058 %ret = add <4 x i32> %odd, %even 2059 ret <4 x i32> %ret 2060} 2061 2062; Do not select if constant is too large 2063define <4 x i32> @pmaddwd_negative2(<8 x i16> %A) { 2064; SSE2-LABEL: pmaddwd_negative2: 2065; SSE2: # %bb.0: 2066; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] 2067; SSE2-NEXT: psrad $16, %xmm1 2068; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7] 2069; SSE2-NEXT: psrad $16, %xmm0 2070; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] 2071; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1,7,42,32] 2072; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3] 2073; SSE2-NEXT: pmuludq %xmm2, %xmm4 2074; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3] 2075; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [32768,4294934528,0,0] 2076; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3] 2077; SSE2-NEXT: pmuludq %xmm2, %xmm6 2078; SSE2-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,2],xmm4[0,2] 2079; SSE2-NEXT: pmuludq %xmm3, %xmm0 2080; SSE2-NEXT: pmuludq %xmm5, %xmm1 2081; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm0[0,2] 2082; SSE2-NEXT: paddd %xmm6, %xmm1 2083; SSE2-NEXT: movdqa %xmm1, %xmm0 2084; SSE2-NEXT: retq 2085; 2086; AVX1-LABEL: pmaddwd_negative2: 2087; AVX1: # %bb.0: 2088; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 2089; AVX1-NEXT: vpmovsxwd %xmm1, %xmm1 2090; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0 2091; AVX1-NEXT: vpmulld {{.*}}(%rip), %xmm0, %xmm0 2092; AVX1-NEXT: vpmulld {{.*}}(%rip), %xmm1, %xmm1 2093; AVX1-NEXT: vphaddd %xmm1, %xmm0, %xmm0 2094; AVX1-NEXT: retq 2095; 2096; AVX256-LABEL: pmaddwd_negative2: 2097; AVX256: # %bb.0: 2098; AVX256-NEXT: vpmovsxwd %xmm0, %ymm0 2099; AVX256-NEXT: vpmulld {{.*}}(%rip), %ymm0, %ymm0 2100; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1 2101; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0 2102; AVX256-NEXT: vzeroupper 2103; AVX256-NEXT: retq 2104 %a = sext <8 x i16> %A to <8 x i32> 2105 %m = mul nsw <8 x i32> %a, <i32 32768, i32 -32768, i32 0, i32 0, i32 1, i32 7, i32 42, i32 32> 2106 %odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 2107 %even = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 2108 %ret = add <4 x i32> %odd, %even 2109 ret <4 x i32> %ret 2110} 2111 2112define <4 x i32> @jumbled_indices4(<8 x i16> %A, <8 x i16> %B) { 2113; SSE2-LABEL: jumbled_indices4: 2114; SSE2: # %bb.0: 2115; SSE2-NEXT: pmaddwd %xmm1, %xmm0 2116; SSE2-NEXT: retq 2117; 2118; AVX-LABEL: jumbled_indices4: 2119; AVX: # %bb.0: 2120; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 2121; AVX-NEXT: retq 2122 %exta = sext <8 x i16> %A to <8 x i32> 2123 %extb = sext <8 x i16> %B to <8 x i32> 2124 %m = mul <8 x i32> %exta, %extb 2125 %sa = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 3, i32 1, i32 5, i32 6> 2126 %sb = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 2, i32 0, i32 4, i32 7> 2127 %a = add <4 x i32> %sa, %sb 2128 ret <4 x i32> %a 2129} 2130 2131define <8 x i32> @jumbled_indices8(<16 x i16> %A, <16 x i16> %B) { 2132; SSE2-LABEL: jumbled_indices8: 2133; SSE2: # %bb.0: 2134; SSE2-NEXT: pmaddwd %xmm2, %xmm0 2135; SSE2-NEXT: pmaddwd %xmm3, %xmm1 2136; SSE2-NEXT: retq 2137; 2138; AVX1-LABEL: jumbled_indices8: 2139; AVX1: # %bb.0: 2140; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 2141; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 2142; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2 2143; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 2144; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 2145; AVX1-NEXT: retq 2146; 2147; AVX256-LABEL: jumbled_indices8: 2148; AVX256: # %bb.0: 2149; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0 2150; AVX256-NEXT: retq 2151 %exta = sext <16 x i16> %A to <16 x i32> 2152 %extb = sext <16 x i16> %B to <16 x i32> 2153 %m = mul <16 x i32> %exta, %extb 2154 %sa = shufflevector <16 x i32> %m, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 7, i32 4, i32 11, i32 8, i32 15, i32 12> 2155 %sb = shufflevector <16 x i32> %m, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 6, i32 5, i32 10, i32 9, i32 14, i32 13> 2156 %a = add <8 x i32> %sa, %sb 2157 ret <8 x i32> %a 2158} 2159 2160define <16 x i32> @jumbled_indices16(<32 x i16> %A, <32 x i16> %B) { 2161; SSE2-LABEL: jumbled_indices16: 2162; SSE2: # %bb.0: 2163; SSE2-NEXT: pmaddwd %xmm4, %xmm0 2164; SSE2-NEXT: pmaddwd %xmm5, %xmm1 2165; SSE2-NEXT: pmaddwd %xmm6, %xmm2 2166; SSE2-NEXT: pmaddwd %xmm7, %xmm3 2167; SSE2-NEXT: retq 2168; 2169; AVX1-LABEL: jumbled_indices16: 2170; AVX1: # %bb.0: 2171; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 2172; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 2173; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm6 2174; AVX1-NEXT: vpmaddwd %xmm6, %xmm4, %xmm4 2175; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6 2176; AVX1-NEXT: vpmaddwd %xmm6, %xmm5, %xmm5 2177; AVX1-NEXT: vpmaddwd %xmm2, %xmm0, %xmm0 2178; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm0 2179; AVX1-NEXT: vpmaddwd %xmm3, %xmm1, %xmm1 2180; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1 2181; AVX1-NEXT: retq 2182; 2183; AVX2-LABEL: jumbled_indices16: 2184; AVX2: # %bb.0: 2185; AVX2-NEXT: vpmaddwd %ymm2, %ymm0, %ymm0 2186; AVX2-NEXT: vpmaddwd %ymm3, %ymm1, %ymm1 2187; AVX2-NEXT: retq 2188; 2189; AVX512F-LABEL: jumbled_indices16: 2190; AVX512F: # %bb.0: 2191; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2 2192; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3 2193; AVX512F-NEXT: vpmaddwd %ymm2, %ymm3, %ymm2 2194; AVX512F-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0 2195; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 2196; AVX512F-NEXT: retq 2197; 2198; AVX512BW-LABEL: jumbled_indices16: 2199; AVX512BW: # %bb.0: 2200; AVX512BW-NEXT: vpmaddwd %zmm1, %zmm0, %zmm0 2201; AVX512BW-NEXT: retq 2202 %exta = sext <32 x i16> %A to <32 x i32> 2203 %extb = sext <32 x i16> %B to <32 x i32> 2204 %m = mul <32 x i32> %exta, %extb 2205 %sa = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 2, i32 0, i32 5, i32 6, i32 11, i32 9, i32 15, i32 12, i32 17, i32 18, i32 20, i32 23, i32 27, i32 24, i32 31, i32 29> 2206 %sb = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 3, i32 1, i32 4, i32 7, i32 10, i32 8, i32 14, i32 13, i32 16, i32 19, i32 21, i32 22, i32 26, i32 25, i32 30, i32 28> 2207 %a = add <16 x i32> %sa, %sb 2208 ret <16 x i32> %a 2209} 2210 2211define <32 x i32> @jumbled_indices32(<64 x i16> %A, <64 x i16> %B) { 2212; SSE2-LABEL: jumbled_indices32: 2213; SSE2: # %bb.0: 2214; SSE2-NEXT: movq %rdi, %rax 2215; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm0 2216; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm1 2217; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm2 2218; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm3 2219; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm4 2220; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm5 2221; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm6 2222; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm7 2223; SSE2-NEXT: movdqa %xmm7, 112(%rdi) 2224; SSE2-NEXT: movdqa %xmm6, 96(%rdi) 2225; SSE2-NEXT: movdqa %xmm5, 80(%rdi) 2226; SSE2-NEXT: movdqa %xmm4, 64(%rdi) 2227; SSE2-NEXT: movdqa %xmm3, 48(%rdi) 2228; SSE2-NEXT: movdqa %xmm2, 32(%rdi) 2229; SSE2-NEXT: movdqa %xmm1, 16(%rdi) 2230; SSE2-NEXT: movdqa %xmm0, (%rdi) 2231; SSE2-NEXT: retq 2232; 2233; AVX1-LABEL: jumbled_indices32: 2234; AVX1: # %bb.0: 2235; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm8 2236; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm9 2237; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm10 2238; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm11 2239; AVX1-NEXT: vextractf128 $1, %ymm7, %xmm12 2240; AVX1-NEXT: vpmaddwd %xmm12, %xmm8, %xmm8 2241; AVX1-NEXT: vextractf128 $1, %ymm6, %xmm12 2242; AVX1-NEXT: vpmaddwd %xmm12, %xmm9, %xmm9 2243; AVX1-NEXT: vextractf128 $1, %ymm5, %xmm12 2244; AVX1-NEXT: vpmaddwd %xmm12, %xmm10, %xmm10 2245; AVX1-NEXT: vextractf128 $1, %ymm4, %xmm12 2246; AVX1-NEXT: vpmaddwd %xmm12, %xmm11, %xmm11 2247; AVX1-NEXT: vpmaddwd %xmm4, %xmm0, %xmm0 2248; AVX1-NEXT: vinsertf128 $1, %xmm11, %ymm0, %ymm0 2249; AVX1-NEXT: vpmaddwd %xmm5, %xmm1, %xmm1 2250; AVX1-NEXT: vinsertf128 $1, %xmm10, %ymm1, %ymm1 2251; AVX1-NEXT: vpmaddwd %xmm6, %xmm2, %xmm2 2252; AVX1-NEXT: vinsertf128 $1, %xmm9, %ymm2, %ymm2 2253; AVX1-NEXT: vpmaddwd %xmm7, %xmm3, %xmm3 2254; AVX1-NEXT: vinsertf128 $1, %xmm8, %ymm3, %ymm3 2255; AVX1-NEXT: retq 2256; 2257; AVX2-LABEL: jumbled_indices32: 2258; AVX2: # %bb.0: 2259; AVX2-NEXT: vpmaddwd %ymm4, %ymm0, %ymm0 2260; AVX2-NEXT: vpmaddwd %ymm5, %ymm1, %ymm1 2261; AVX2-NEXT: vpmaddwd %ymm6, %ymm2, %ymm2 2262; AVX2-NEXT: vpmaddwd %ymm7, %ymm3, %ymm3 2263; AVX2-NEXT: retq 2264; 2265; AVX512F-LABEL: jumbled_indices32: 2266; AVX512F: # %bb.0: 2267; AVX512F-NEXT: vpmaddwd %ymm5, %ymm1, %ymm1 2268; AVX512F-NEXT: vpmaddwd %ymm4, %ymm0, %ymm0 2269; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 2270; AVX512F-NEXT: vpmaddwd %ymm7, %ymm3, %ymm1 2271; AVX512F-NEXT: vpmaddwd %ymm6, %ymm2, %ymm2 2272; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1 2273; AVX512F-NEXT: retq 2274; 2275; AVX512BW-LABEL: jumbled_indices32: 2276; AVX512BW: # %bb.0: 2277; AVX512BW-NEXT: vpmaddwd %zmm2, %zmm0, %zmm0 2278; AVX512BW-NEXT: vpmaddwd %zmm3, %zmm1, %zmm1 2279; AVX512BW-NEXT: retq 2280 %exta = sext <64 x i16> %A to <64 x i32> 2281 %extb = sext <64 x i16> %B to <64 x i32> 2282 %m = mul <64 x i32> %exta, %extb 2283 %sa = shufflevector <64 x i32> %m, <64 x i32> undef, <32 x i32> <i32 1, i32 2, i32 6, i32 5, i32 10, i32 8, i32 14, i32 12, i32 19, i32 17, i32 22, i32 20, i32 25, i32 27, i32 30, i32 28, i32 32, i32 34, i32 37, i32 38, i32 41, i32 43, i32 45, i32 47, i32 50, i32 48, i32 52, i32 54, i32 59, i32 56, i32 61, i32 63> 2284 %sb = shufflevector <64 x i32> %m, <64 x i32> undef, <32 x i32> <i32 0, i32 3, i32 7, i32 4, i32 11, i32 9, i32 15, i32 13, i32 18, i32 16, i32 23, i32 21, i32 24, i32 26, i32 31, i32 29, i32 33, i32 35, i32 36, i32 39, i32 40, i32 42, i32 44, i32 46, i32 51, i32 49, i32 53, i32 55, i32 58, i32 57, i32 60, i32 62> 2285 %a = add <32 x i32> %sa, %sb 2286 ret <32 x i32> %a 2287} 2288 2289; NOTE: We're testing with loads because ABI lowering creates a concat_vectors that extract_vector_elt creation can see through. 2290; This would require the combine to recreate the concat_vectors. 2291define <4 x i32> @pmaddwd_128(<8 x i16>* %Aptr, <8 x i16>* %Bptr) { 2292; SSE2-LABEL: pmaddwd_128: 2293; SSE2: # %bb.0: 2294; SSE2-NEXT: movdqa (%rdi), %xmm0 2295; SSE2-NEXT: pmaddwd (%rsi), %xmm0 2296; SSE2-NEXT: retq 2297; 2298; AVX-LABEL: pmaddwd_128: 2299; AVX: # %bb.0: 2300; AVX-NEXT: vmovdqa (%rdi), %xmm0 2301; AVX-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0 2302; AVX-NEXT: retq 2303 %A = load <8 x i16>, <8 x i16>* %Aptr 2304 %B = load <8 x i16>, <8 x i16>* %Bptr 2305 %A_even = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 2306 %A_odd = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 2307 %B_even = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 2308 %B_odd = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 2309 %A_even_ext = sext <4 x i16> %A_even to <4 x i32> 2310 %B_even_ext = sext <4 x i16> %B_even to <4 x i32> 2311 %A_odd_ext = sext <4 x i16> %A_odd to <4 x i32> 2312 %B_odd_ext = sext <4 x i16> %B_odd to <4 x i32> 2313 %even_mul = mul <4 x i32> %A_even_ext, %B_even_ext 2314 %odd_mul = mul <4 x i32> %A_odd_ext, %B_odd_ext 2315 %add = add <4 x i32> %even_mul, %odd_mul 2316 ret <4 x i32> %add 2317} 2318 2319define <8 x i32> @pmaddwd_256(<16 x i16>* %Aptr, <16 x i16>* %Bptr) { 2320; SSE2-LABEL: pmaddwd_256: 2321; SSE2: # %bb.0: 2322; SSE2-NEXT: movdqa (%rdi), %xmm0 2323; SSE2-NEXT: movdqa 16(%rdi), %xmm1 2324; SSE2-NEXT: pmaddwd (%rsi), %xmm0 2325; SSE2-NEXT: pmaddwd 16(%rsi), %xmm1 2326; SSE2-NEXT: retq 2327; 2328; AVX1-LABEL: pmaddwd_256: 2329; AVX1: # %bb.0: 2330; AVX1-NEXT: vmovdqa (%rdi), %xmm0 2331; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 2332; AVX1-NEXT: vpmaddwd 16(%rsi), %xmm1, %xmm1 2333; AVX1-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0 2334; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 2335; AVX1-NEXT: retq 2336; 2337; AVX256-LABEL: pmaddwd_256: 2338; AVX256: # %bb.0: 2339; AVX256-NEXT: vmovdqa (%rdi), %ymm0 2340; AVX256-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0 2341; AVX256-NEXT: retq 2342 %A = load <16 x i16>, <16 x i16>* %Aptr 2343 %B = load <16 x i16>, <16 x i16>* %Bptr 2344 %A_even = shufflevector <16 x i16> %A, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14> 2345 %A_odd = shufflevector <16 x i16> %A, <16 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15> 2346 %B_even = shufflevector <16 x i16> %B, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14> 2347 %B_odd = shufflevector <16 x i16> %B, <16 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15> 2348 %A_even_ext = sext <8 x i16> %A_even to <8 x i32> 2349 %B_even_ext = sext <8 x i16> %B_even to <8 x i32> 2350 %A_odd_ext = sext <8 x i16> %A_odd to <8 x i32> 2351 %B_odd_ext = sext <8 x i16> %B_odd to <8 x i32> 2352 %even_mul = mul <8 x i32> %A_even_ext, %B_even_ext 2353 %odd_mul = mul <8 x i32> %A_odd_ext, %B_odd_ext 2354 %add = add <8 x i32> %even_mul, %odd_mul 2355 ret <8 x i32> %add 2356} 2357 2358define <16 x i32> @pmaddwd_512(<32 x i16>* %Aptr, <32 x i16>* %Bptr) { 2359; SSE2-LABEL: pmaddwd_512: 2360; SSE2: # %bb.0: 2361; SSE2-NEXT: movdqa (%rdi), %xmm0 2362; SSE2-NEXT: movdqa 16(%rdi), %xmm1 2363; SSE2-NEXT: movdqa 32(%rdi), %xmm2 2364; SSE2-NEXT: movdqa 48(%rdi), %xmm3 2365; SSE2-NEXT: pmaddwd (%rsi), %xmm0 2366; SSE2-NEXT: pmaddwd 16(%rsi), %xmm1 2367; SSE2-NEXT: pmaddwd 32(%rsi), %xmm2 2368; SSE2-NEXT: pmaddwd 48(%rsi), %xmm3 2369; SSE2-NEXT: retq 2370; 2371; AVX1-LABEL: pmaddwd_512: 2372; AVX1: # %bb.0: 2373; AVX1-NEXT: vmovdqa (%rdi), %xmm0 2374; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 2375; AVX1-NEXT: vmovdqa 32(%rdi), %xmm2 2376; AVX1-NEXT: vmovdqa 48(%rdi), %xmm3 2377; AVX1-NEXT: vpmaddwd 16(%rsi), %xmm1, %xmm1 2378; AVX1-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0 2379; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 2380; AVX1-NEXT: vpmaddwd 48(%rsi), %xmm3, %xmm1 2381; AVX1-NEXT: vpmaddwd 32(%rsi), %xmm2, %xmm2 2382; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 2383; AVX1-NEXT: retq 2384; 2385; AVX2-LABEL: pmaddwd_512: 2386; AVX2: # %bb.0: 2387; AVX2-NEXT: vmovdqa (%rdi), %ymm0 2388; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1 2389; AVX2-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0 2390; AVX2-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1 2391; AVX2-NEXT: retq 2392; 2393; AVX512F-LABEL: pmaddwd_512: 2394; AVX512F: # %bb.0: 2395; AVX512F-NEXT: vmovdqa (%rdi), %ymm0 2396; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1 2397; AVX512F-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1 2398; AVX512F-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0 2399; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 2400; AVX512F-NEXT: retq 2401; 2402; AVX512BW-LABEL: pmaddwd_512: 2403; AVX512BW: # %bb.0: 2404; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0 2405; AVX512BW-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0 2406; AVX512BW-NEXT: retq 2407 %A = load <32 x i16>, <32 x i16>* %Aptr 2408 %B = load <32 x i16>, <32 x i16>* %Bptr 2409 %A_even = shufflevector <32 x i16> %A, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30> 2410 %A_odd = shufflevector <32 x i16> %A, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31> 2411 %B_even = shufflevector <32 x i16> %B, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30> 2412 %B_odd = shufflevector <32 x i16> %B, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31> 2413 %A_even_ext = sext <16 x i16> %A_even to <16 x i32> 2414 %B_even_ext = sext <16 x i16> %B_even to <16 x i32> 2415 %A_odd_ext = sext <16 x i16> %A_odd to <16 x i32> 2416 %B_odd_ext = sext <16 x i16> %B_odd to <16 x i32> 2417 %even_mul = mul <16 x i32> %A_even_ext, %B_even_ext 2418 %odd_mul = mul <16 x i32> %A_odd_ext, %B_odd_ext 2419 %add = add <16 x i32> %even_mul, %odd_mul 2420 ret <16 x i32> %add 2421} 2422 2423define <32 x i32> @pmaddwd_1024(<64 x i16>* %Aptr, <64 x i16>* %Bptr) { 2424; SSE2-LABEL: pmaddwd_1024: 2425; SSE2: # %bb.0: 2426; SSE2-NEXT: movq %rdi, %rax 2427; SSE2-NEXT: movdqa (%rsi), %xmm0 2428; SSE2-NEXT: movdqa 16(%rsi), %xmm1 2429; SSE2-NEXT: movdqa 32(%rsi), %xmm2 2430; SSE2-NEXT: movdqa 48(%rsi), %xmm3 2431; SSE2-NEXT: pmaddwd (%rdx), %xmm0 2432; SSE2-NEXT: pmaddwd 16(%rdx), %xmm1 2433; SSE2-NEXT: pmaddwd 32(%rdx), %xmm2 2434; SSE2-NEXT: pmaddwd 48(%rdx), %xmm3 2435; SSE2-NEXT: movdqa 64(%rsi), %xmm4 2436; SSE2-NEXT: pmaddwd 64(%rdx), %xmm4 2437; SSE2-NEXT: movdqa 80(%rsi), %xmm5 2438; SSE2-NEXT: pmaddwd 80(%rdx), %xmm5 2439; SSE2-NEXT: movdqa 96(%rsi), %xmm6 2440; SSE2-NEXT: pmaddwd 96(%rdx), %xmm6 2441; SSE2-NEXT: movdqa 112(%rsi), %xmm7 2442; SSE2-NEXT: pmaddwd 112(%rdx), %xmm7 2443; SSE2-NEXT: movdqa %xmm7, 112(%rdi) 2444; SSE2-NEXT: movdqa %xmm6, 96(%rdi) 2445; SSE2-NEXT: movdqa %xmm5, 80(%rdi) 2446; SSE2-NEXT: movdqa %xmm4, 64(%rdi) 2447; SSE2-NEXT: movdqa %xmm3, 48(%rdi) 2448; SSE2-NEXT: movdqa %xmm2, 32(%rdi) 2449; SSE2-NEXT: movdqa %xmm1, 16(%rdi) 2450; SSE2-NEXT: movdqa %xmm0, (%rdi) 2451; SSE2-NEXT: retq 2452; 2453; AVX1-LABEL: pmaddwd_1024: 2454; AVX1: # %bb.0: 2455; AVX1-NEXT: vmovdqa (%rdi), %xmm0 2456; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 2457; AVX1-NEXT: vmovdqa 32(%rdi), %xmm2 2458; AVX1-NEXT: vmovdqa 48(%rdi), %xmm3 2459; AVX1-NEXT: vpmaddwd 16(%rsi), %xmm1, %xmm1 2460; AVX1-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0 2461; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 2462; AVX1-NEXT: vpmaddwd 48(%rsi), %xmm3, %xmm1 2463; AVX1-NEXT: vpmaddwd 32(%rsi), %xmm2, %xmm2 2464; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 2465; AVX1-NEXT: vmovdqa 80(%rdi), %xmm2 2466; AVX1-NEXT: vpmaddwd 80(%rsi), %xmm2, %xmm2 2467; AVX1-NEXT: vmovdqa 64(%rdi), %xmm3 2468; AVX1-NEXT: vpmaddwd 64(%rsi), %xmm3, %xmm3 2469; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 2470; AVX1-NEXT: vmovdqa 112(%rdi), %xmm3 2471; AVX1-NEXT: vpmaddwd 112(%rsi), %xmm3, %xmm3 2472; AVX1-NEXT: vmovdqa 96(%rdi), %xmm4 2473; AVX1-NEXT: vpmaddwd 96(%rsi), %xmm4, %xmm4 2474; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3 2475; AVX1-NEXT: retq 2476; 2477; AVX2-LABEL: pmaddwd_1024: 2478; AVX2: # %bb.0: 2479; AVX2-NEXT: vmovdqa (%rdi), %ymm0 2480; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1 2481; AVX2-NEXT: vmovdqa 64(%rdi), %ymm2 2482; AVX2-NEXT: vmovdqa 96(%rdi), %ymm3 2483; AVX2-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0 2484; AVX2-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1 2485; AVX2-NEXT: vpmaddwd 64(%rsi), %ymm2, %ymm2 2486; AVX2-NEXT: vpmaddwd 96(%rsi), %ymm3, %ymm3 2487; AVX2-NEXT: retq 2488; 2489; AVX512F-LABEL: pmaddwd_1024: 2490; AVX512F: # %bb.0: 2491; AVX512F-NEXT: vmovdqa (%rdi), %ymm0 2492; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1 2493; AVX512F-NEXT: vmovdqa 64(%rdi), %ymm2 2494; AVX512F-NEXT: vmovdqa 96(%rdi), %ymm3 2495; AVX512F-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1 2496; AVX512F-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0 2497; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 2498; AVX512F-NEXT: vpmaddwd 96(%rsi), %ymm3, %ymm1 2499; AVX512F-NEXT: vpmaddwd 64(%rsi), %ymm2, %ymm2 2500; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1 2501; AVX512F-NEXT: retq 2502; 2503; AVX512BW-LABEL: pmaddwd_1024: 2504; AVX512BW: # %bb.0: 2505; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0 2506; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1 2507; AVX512BW-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0 2508; AVX512BW-NEXT: vpmaddwd 64(%rsi), %zmm1, %zmm1 2509; AVX512BW-NEXT: retq 2510 %A = load <64 x i16>, <64 x i16>* %Aptr 2511 %B = load <64 x i16>, <64 x i16>* %Bptr 2512 %A_even = shufflevector <64 x i16> %A, <64 x i16> undef, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62> 2513 %A_odd = shufflevector <64 x i16> %A, <64 x i16> undef, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63> 2514 %B_even = shufflevector <64 x i16> %B, <64 x i16> undef, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62> 2515 %B_odd = shufflevector <64 x i16> %B, <64 x i16> undef, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63> 2516 %A_even_ext = sext <32 x i16> %A_even to <32 x i32> 2517 %B_even_ext = sext <32 x i16> %B_even to <32 x i32> 2518 %A_odd_ext = sext <32 x i16> %A_odd to <32 x i32> 2519 %B_odd_ext = sext <32 x i16> %B_odd to <32 x i32> 2520 %even_mul = mul <32 x i32> %A_even_ext, %B_even_ext 2521 %odd_mul = mul <32 x i32> %A_odd_ext, %B_odd_ext 2522 %add = add <32 x i32> %even_mul, %odd_mul 2523 ret <32 x i32> %add 2524} 2525 2526define <4 x i32> @pmaddwd_commuted_mul(<8 x i16>* %Aptr, <8 x i16>* %Bptr) { 2527; SSE2-LABEL: pmaddwd_commuted_mul: 2528; SSE2: # %bb.0: 2529; SSE2-NEXT: movdqa (%rdi), %xmm0 2530; SSE2-NEXT: pmaddwd (%rsi), %xmm0 2531; SSE2-NEXT: retq 2532; 2533; AVX-LABEL: pmaddwd_commuted_mul: 2534; AVX: # %bb.0: 2535; AVX-NEXT: vmovdqa (%rdi), %xmm0 2536; AVX-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0 2537; AVX-NEXT: retq 2538 %A = load <8 x i16>, <8 x i16>* %Aptr 2539 %B = load <8 x i16>, <8 x i16>* %Bptr 2540 %A_even = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 2541 %A_odd = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 2542 %B_even = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 2543 %B_odd = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 2544 %A_even_ext = sext <4 x i16> %A_even to <4 x i32> 2545 %B_even_ext = sext <4 x i16> %B_even to <4 x i32> 2546 %A_odd_ext = sext <4 x i16> %A_odd to <4 x i32> 2547 %B_odd_ext = sext <4 x i16> %B_odd to <4 x i32> 2548 %even_mul = mul <4 x i32> %A_even_ext, %B_even_ext 2549 %odd_mul = mul <4 x i32> %B_odd_ext, %A_odd_ext ; Different order than previous mul 2550 %add = add <4 x i32> %even_mul, %odd_mul 2551 ret <4 x i32> %add 2552} 2553 2554define <4 x i32> @pmaddwd_swapped_indices(<8 x i16>* %Aptr, <8 x i16>* %Bptr) { 2555; SSE2-LABEL: pmaddwd_swapped_indices: 2556; SSE2: # %bb.0: 2557; SSE2-NEXT: movdqa (%rdi), %xmm0 2558; SSE2-NEXT: pmaddwd (%rsi), %xmm0 2559; SSE2-NEXT: retq 2560; 2561; AVX-LABEL: pmaddwd_swapped_indices: 2562; AVX: # %bb.0: 2563; AVX-NEXT: vmovdqa (%rdi), %xmm0 2564; AVX-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0 2565; AVX-NEXT: retq 2566 %A = load <8 x i16>, <8 x i16>* %Aptr 2567 %B = load <8 x i16>, <8 x i16>* %Bptr 2568 %A_even = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 1, i32 2, i32 5, i32 6> ; indices aren't all even 2569 %A_odd = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 7> ; indices aren't all odd 2570 %B_even = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 2, i32 5, i32 6> ; same indices as A 2571 %B_odd = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 7> ; same indices as A 2572 %A_even_ext = sext <4 x i16> %A_even to <4 x i32> 2573 %B_even_ext = sext <4 x i16> %B_even to <4 x i32> 2574 %A_odd_ext = sext <4 x i16> %A_odd to <4 x i32> 2575 %B_odd_ext = sext <4 x i16> %B_odd to <4 x i32> 2576 %even_mul = mul <4 x i32> %A_even_ext, %B_even_ext 2577 %odd_mul = mul <4 x i32> %A_odd_ext, %B_odd_ext 2578 %add = add <4 x i32> %even_mul, %odd_mul 2579 ret <4 x i32> %add 2580} 2581 2582; Negative test were indices aren't paired properly 2583define <4 x i32> @pmaddwd_bad_indices(<8 x i16>* %Aptr, <8 x i16>* %Bptr) { 2584; SSE2-LABEL: pmaddwd_bad_indices: 2585; SSE2: # %bb.0: 2586; SSE2-NEXT: movdqa (%rdi), %xmm0 2587; SSE2-NEXT: movdqa (%rsi), %xmm1 2588; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm0[2,1,2,3,4,5,6,7] 2589; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7] 2590; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] 2591; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,0,3,2,4,5,6,7] 2592; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] 2593; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] 2594; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] 2595; SSE2-NEXT: pshuflw {{.*#+}} xmm3 = xmm1[0,2,2,3,4,5,6,7] 2596; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7] 2597; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] 2598; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] 2599; SSE2-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] 2600; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] 2601; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7] 2602; SSE2-NEXT: movdqa %xmm2, %xmm4 2603; SSE2-NEXT: pmulhw %xmm3, %xmm4 2604; SSE2-NEXT: pmullw %xmm3, %xmm2 2605; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3] 2606; SSE2-NEXT: movdqa %xmm0, %xmm3 2607; SSE2-NEXT: pmulhw %xmm1, %xmm3 2608; SSE2-NEXT: pmullw %xmm1, %xmm0 2609; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] 2610; SSE2-NEXT: paddd %xmm2, %xmm0 2611; SSE2-NEXT: retq 2612; 2613; AVX-LABEL: pmaddwd_bad_indices: 2614; AVX: # %bb.0: 2615; AVX-NEXT: vmovdqa (%rdi), %xmm0 2616; AVX-NEXT: vmovdqa (%rsi), %xmm1 2617; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,3,4,5,10,11,12,13,12,13,10,11,12,13,14,15] 2618; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,6,7,8,9,14,15,8,9,14,15,12,13,14,15] 2619; AVX-NEXT: vpshufb {{.*#+}} xmm3 = xmm1[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] 2620; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15] 2621; AVX-NEXT: vpmovsxwd %xmm2, %xmm2 2622; AVX-NEXT: vpmovsxwd %xmm3, %xmm3 2623; AVX-NEXT: vpmulld %xmm3, %xmm2, %xmm2 2624; AVX-NEXT: vpmovsxwd %xmm0, %xmm0 2625; AVX-NEXT: vpmovsxwd %xmm1, %xmm1 2626; AVX-NEXT: vpmulld %xmm1, %xmm0, %xmm0 2627; AVX-NEXT: vpaddd %xmm0, %xmm2, %xmm0 2628; AVX-NEXT: retq 2629 %A = load <8 x i16>, <8 x i16>* %Aptr 2630 %B = load <8 x i16>, <8 x i16>* %Bptr 2631 %A_even = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 1, i32 2, i32 5, i32 6> 2632 %A_odd = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 7> 2633 %B_even = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> ; different indices than A 2634 %B_odd = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> ; different indices than A 2635 %A_even_ext = sext <4 x i16> %A_even to <4 x i32> 2636 %B_even_ext = sext <4 x i16> %B_even to <4 x i32> 2637 %A_odd_ext = sext <4 x i16> %A_odd to <4 x i32> 2638 %B_odd_ext = sext <4 x i16> %B_odd to <4 x i32> 2639 %even_mul = mul <4 x i32> %A_even_ext, %B_even_ext 2640 %odd_mul = mul <4 x i32> %A_odd_ext, %B_odd_ext 2641 %add = add <4 x i32> %even_mul, %odd_mul 2642 ret <4 x i32> %add 2643} 2644 2645; This test contains two multiplies joined by an add. The result of that add is then reduced to a single element. 2646; SelectionDAGBuilder should tag the joining add as a vector reduction. We need to recognize that both sides can use pmaddwd 2647define i32 @madd_double_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>* %arg2, <8 x i16>* %arg3) { 2648; SSE2-LABEL: madd_double_reduction: 2649; SSE2: # %bb.0: 2650; SSE2-NEXT: movdqu (%rdi), %xmm0 2651; SSE2-NEXT: movdqu (%rsi), %xmm1 2652; SSE2-NEXT: pmaddwd %xmm0, %xmm1 2653; SSE2-NEXT: movdqu (%rdx), %xmm0 2654; SSE2-NEXT: movdqu (%rcx), %xmm2 2655; SSE2-NEXT: pmaddwd %xmm0, %xmm2 2656; SSE2-NEXT: paddd %xmm1, %xmm2 2657; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1] 2658; SSE2-NEXT: paddd %xmm2, %xmm0 2659; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 2660; SSE2-NEXT: paddd %xmm0, %xmm1 2661; SSE2-NEXT: movd %xmm1, %eax 2662; SSE2-NEXT: retq 2663; 2664; AVX-LABEL: madd_double_reduction: 2665; AVX: # %bb.0: 2666; AVX-NEXT: vmovdqu (%rdi), %xmm0 2667; AVX-NEXT: vmovdqu (%rdx), %xmm1 2668; AVX-NEXT: vpmaddwd (%rcx), %xmm1, %xmm1 2669; AVX-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0 2670; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 2671; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 2672; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 2673; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 2674; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 2675; AVX-NEXT: vmovd %xmm0, %eax 2676; AVX-NEXT: retq 2677 %tmp = load <8 x i16>, <8 x i16>* %arg, align 1 2678 %tmp6 = load <8 x i16>, <8 x i16>* %arg1, align 1 2679 %tmp7 = sext <8 x i16> %tmp to <8 x i32> 2680 %tmp17 = sext <8 x i16> %tmp6 to <8 x i32> 2681 %tmp19 = mul nsw <8 x i32> %tmp7, %tmp17 2682 %tmp20 = load <8 x i16>, <8 x i16>* %arg2, align 1 2683 %tmp21 = load <8 x i16>, <8 x i16>* %arg3, align 1 2684 %tmp22 = sext <8 x i16> %tmp20 to <8 x i32> 2685 %tmp23 = sext <8 x i16> %tmp21 to <8 x i32> 2686 %tmp25 = mul nsw <8 x i32> %tmp22, %tmp23 2687 %tmp26 = add nuw nsw <8 x i32> %tmp25, %tmp19 2688 %tmp29 = shufflevector <8 x i32> %tmp26, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef> 2689 %tmp30 = add <8 x i32> %tmp26, %tmp29 2690 %tmp31 = shufflevector <8 x i32> %tmp30, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 2691 %tmp32 = add <8 x i32> %tmp30, %tmp31 2692 %tmp33 = shufflevector <8 x i32> %tmp32, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 2693 %tmp34 = add <8 x i32> %tmp32, %tmp33 2694 %tmp35 = extractelement <8 x i32> %tmp34, i64 0 2695 ret i32 %tmp35 2696} 2697 2698define i32 @madd_quad_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>* %arg2, <8 x i16>* %arg3, <8 x i16>* %arg4, <8 x i16>* %arg5, <8 x i16>* %arg6, <8 x i16>* %arg7) { 2699; SSE2-LABEL: madd_quad_reduction: 2700; SSE2: # %bb.0: 2701; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r10 2702; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax 2703; SSE2-NEXT: movdqu (%rdi), %xmm0 2704; SSE2-NEXT: movdqu (%rsi), %xmm1 2705; SSE2-NEXT: pmaddwd %xmm0, %xmm1 2706; SSE2-NEXT: movdqu (%rdx), %xmm0 2707; SSE2-NEXT: movdqu (%rcx), %xmm2 2708; SSE2-NEXT: pmaddwd %xmm0, %xmm2 2709; SSE2-NEXT: movdqu (%r8), %xmm0 2710; SSE2-NEXT: movdqu (%r9), %xmm3 2711; SSE2-NEXT: pmaddwd %xmm0, %xmm3 2712; SSE2-NEXT: paddd %xmm2, %xmm3 2713; SSE2-NEXT: movdqu (%rax), %xmm0 2714; SSE2-NEXT: movdqu (%r10), %xmm2 2715; SSE2-NEXT: pmaddwd %xmm0, %xmm2 2716; SSE2-NEXT: paddd %xmm3, %xmm2 2717; SSE2-NEXT: paddd %xmm1, %xmm2 2718; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1] 2719; SSE2-NEXT: paddd %xmm2, %xmm0 2720; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 2721; SSE2-NEXT: paddd %xmm0, %xmm1 2722; SSE2-NEXT: movd %xmm1, %eax 2723; SSE2-NEXT: retq 2724; 2725; AVX-LABEL: madd_quad_reduction: 2726; AVX: # %bb.0: 2727; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r10 2728; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax 2729; AVX-NEXT: vmovdqu (%rdi), %xmm0 2730; AVX-NEXT: vmovdqu (%rdx), %xmm1 2731; AVX-NEXT: vpmaddwd (%rcx), %xmm1, %xmm1 2732; AVX-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0 2733; AVX-NEXT: vmovdqu (%r8), %xmm2 2734; AVX-NEXT: vpmaddwd (%r9), %xmm2, %xmm2 2735; AVX-NEXT: vpaddd %xmm2, %xmm1, %xmm1 2736; AVX-NEXT: vmovdqu (%rax), %xmm2 2737; AVX-NEXT: vpmaddwd (%r10), %xmm2, %xmm2 2738; AVX-NEXT: vpaddd %xmm2, %xmm1, %xmm1 2739; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 2740; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 2741; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 2742; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 2743; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 2744; AVX-NEXT: vmovd %xmm0, %eax 2745; AVX-NEXT: retq 2746 %tmp = load <8 x i16>, <8 x i16>* %arg, align 1 2747 %tmp6 = load <8 x i16>, <8 x i16>* %arg1, align 1 2748 %tmp7 = sext <8 x i16> %tmp to <8 x i32> 2749 %tmp17 = sext <8 x i16> %tmp6 to <8 x i32> 2750 %tmp19 = mul nsw <8 x i32> %tmp7, %tmp17 2751 %tmp20 = load <8 x i16>, <8 x i16>* %arg2, align 1 2752 %tmp21 = load <8 x i16>, <8 x i16>* %arg3, align 1 2753 %tmp22 = sext <8 x i16> %tmp20 to <8 x i32> 2754 %tmp23 = sext <8 x i16> %tmp21 to <8 x i32> 2755 %tmp25 = mul nsw <8 x i32> %tmp22, %tmp23 2756 %tmp26 = add nuw nsw <8 x i32> %tmp25, %tmp19 2757 2758 %tmp40 = load <8 x i16>, <8 x i16>* %arg4, align 1 2759 %tmp41 = load <8 x i16>, <8 x i16>* %arg5, align 1 2760 %tmp42 = sext <8 x i16> %tmp40 to <8 x i32> 2761 %tmp43 = sext <8 x i16> %tmp41 to <8 x i32> 2762 %tmp45 = mul nsw <8 x i32> %tmp42, %tmp43 2763 %tmp56 = add nuw nsw <8 x i32> %tmp26, %tmp45 2764 2765 %tmp50 = load <8 x i16>, <8 x i16>* %arg6, align 1 2766 %tmp51 = load <8 x i16>, <8 x i16>* %arg7, align 1 2767 %tmp52 = sext <8 x i16> %tmp50 to <8 x i32> 2768 %tmp53 = sext <8 x i16> %tmp51 to <8 x i32> 2769 %tmp55 = mul nsw <8 x i32> %tmp52, %tmp53 2770 %tmp57 = add nuw nsw <8 x i32> %tmp55, %tmp56 2771 2772 %tmp29 = shufflevector <8 x i32> %tmp57, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef> 2773 %tmp30 = add <8 x i32> %tmp57, %tmp29 2774 %tmp31 = shufflevector <8 x i32> %tmp30, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 2775 %tmp32 = add <8 x i32> %tmp30, %tmp31 2776 %tmp33 = shufflevector <8 x i32> %tmp32, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 2777 %tmp34 = add <8 x i32> %tmp32, %tmp33 2778 %tmp35 = extractelement <8 x i32> %tmp34, i64 0 2779 ret i32 %tmp35 2780} 2781 2782define i64 @sum_and_sum_of_squares(i8* %a, i32 %n) { 2783; SSE2-LABEL: sum_and_sum_of_squares: 2784; SSE2: # %bb.0: # %entry 2785; SSE2-NEXT: movl %esi, %eax 2786; SSE2-NEXT: pxor %xmm0, %xmm0 2787; SSE2-NEXT: pxor %xmm1, %xmm1 2788; SSE2-NEXT: pxor %xmm2, %xmm2 2789; SSE2-NEXT: pxor %xmm3, %xmm3 2790; SSE2-NEXT: .p2align 4, 0x90 2791; SSE2-NEXT: .LBB33_1: # %vector.body 2792; SSE2-NEXT: # =>This Inner Loop Header: Depth=1 2793; SSE2-NEXT: movq {{.*#+}} xmm4 = mem[0],zero 2794; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 2795; SSE2-NEXT: movdqa %xmm4, %xmm5 2796; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3] 2797; SSE2-NEXT: paddd %xmm5, %xmm2 2798; SSE2-NEXT: movdqa %xmm4, %xmm5 2799; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7] 2800; SSE2-NEXT: paddd %xmm5, %xmm3 2801; SSE2-NEXT: pmaddwd %xmm4, %xmm4 2802; SSE2-NEXT: paddd %xmm4, %xmm1 2803; SSE2-NEXT: addq $8, %rdi 2804; SSE2-NEXT: addq $-8, %rax 2805; SSE2-NEXT: jne .LBB33_1 2806; SSE2-NEXT: # %bb.2: # %middle.block 2807; SSE2-NEXT: paddd %xmm0, %xmm1 2808; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] 2809; SSE2-NEXT: paddd %xmm1, %xmm0 2810; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 2811; SSE2-NEXT: paddd %xmm0, %xmm1 2812; SSE2-NEXT: movd %xmm1, %eax 2813; SSE2-NEXT: retq 2814; 2815; AVX1-LABEL: sum_and_sum_of_squares: 2816; AVX1: # %bb.0: # %entry 2817; AVX1-NEXT: movl %esi, %eax 2818; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0 2819; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 2820; AVX1-NEXT: .p2align 4, 0x90 2821; AVX1-NEXT: .LBB33_1: # %vector.body 2822; AVX1-NEXT: # =>This Inner Loop Header: Depth=1 2823; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero 2824; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero 2825; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 2826; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm4 2827; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm1 2828; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1 2829; AVX1-NEXT: vpmaddwd %xmm2, %xmm2, %xmm2 2830; AVX1-NEXT: vpmaddwd %xmm3, %xmm3, %xmm3 2831; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 2832; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3 2833; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0 2834; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0 2835; AVX1-NEXT: addq $8, %rdi 2836; AVX1-NEXT: addq $-8, %rax 2837; AVX1-NEXT: jne .LBB33_1 2838; AVX1-NEXT: # %bb.2: # %middle.block 2839; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 2840; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 2841; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 2842; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 2843; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 2844; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 2845; AVX1-NEXT: vmovd %xmm0, %eax 2846; AVX1-NEXT: vzeroupper 2847; AVX1-NEXT: retq 2848; 2849; AVX256-LABEL: sum_and_sum_of_squares: 2850; AVX256: # %bb.0: # %entry 2851; AVX256-NEXT: movl %esi, %eax 2852; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0 2853; AVX256-NEXT: vpxor %xmm1, %xmm1, %xmm1 2854; AVX256-NEXT: .p2align 4, 0x90 2855; AVX256-NEXT: .LBB33_1: # %vector.body 2856; AVX256-NEXT: # =>This Inner Loop Header: Depth=1 2857; AVX256-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 2858; AVX256-NEXT: vpaddd %ymm1, %ymm2, %ymm1 2859; AVX256-NEXT: vpmaddwd %ymm2, %ymm2, %ymm2 2860; AVX256-NEXT: vpaddd %ymm0, %ymm2, %ymm0 2861; AVX256-NEXT: addq $8, %rdi 2862; AVX256-NEXT: addq $-8, %rax 2863; AVX256-NEXT: jne .LBB33_1 2864; AVX256-NEXT: # %bb.2: # %middle.block 2865; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1 2866; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0 2867; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] 2868; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0 2869; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] 2870; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0 2871; AVX256-NEXT: vmovd %xmm0, %eax 2872; AVX256-NEXT: vzeroupper 2873; AVX256-NEXT: retq 2874entry: 2875 %0 = zext i32 %n to i64 2876 br label %vector.body 2877 2878vector.body: 2879 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] 2880 %vec.phi = phi <8 x i32> [ %6, %vector.body ], [ zeroinitializer, %entry ] 2881 %sum.phi = phi <8 x i32> [ %4, %vector.body ], [ zeroinitializer, %entry ] 2882 %1 = getelementptr inbounds i8, i8* %a, i64 %index 2883 %2 = bitcast i8* %1 to <8 x i8>* 2884 %wide.load = load <8 x i8>, <8 x i8>* %2, align 1 2885 %3 = zext <8 x i8> %wide.load to <8 x i32> 2886 %4 = add nsw <8 x i32> %3, %sum.phi 2887 %5 = mul nsw <8 x i32> %3, %3 2888 %6 = add nsw <8 x i32> %5, %vec.phi 2889 %index.next = add i64 %index, 8 2890 %7 = icmp eq i64 %index.next, %0 2891 br i1 %7, label %middle.block, label %vector.body 2892 2893middle.block: 2894 %rdx.shuf35 = shufflevector <8 x i32> %4, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef> 2895 %bin.rdx36 = add <8 x i32> %4, %rdx.shuf35 2896 %rdx.shuf37 = shufflevector <8 x i32> %bin.rdx36, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 2897 %bin.rdx38 = add <8 x i32> %bin.rdx36, %rdx.shuf37 2898 %rdx.shuf39 = shufflevector <8 x i32> %bin.rdx38, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 2899 %bin.rdx40 = add <8 x i32> %bin.rdx38, %rdx.shuf39 2900 %8 = extractelement <8 x i32> %bin.rdx40, i32 0 2901 %rdx.shuf = shufflevector <8 x i32> %6, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef> 2902 %bin.rdx = add <8 x i32> %6, %rdx.shuf 2903 %rdx.shuf31 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 2904 %bin.rdx32 = add <8 x i32> %bin.rdx, %rdx.shuf31 2905 %rdx.shuf33 = shufflevector <8 x i32> %bin.rdx32, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 2906 %bin.rdx34 = add <8 x i32> %bin.rdx32, %rdx.shuf33 2907 %9 = extractelement <8 x i32> %bin.rdx34, i32 0 2908 %tmp = zext i32 %8 to i64 2909 %tmp28 = shl nuw i64 %tmp, 32 2910 %tmp29 = zext i32 %9 to i64 2911 ret i64 %tmp29 2912} 2913