1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41 4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F 6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW 7 8define <16 x i8> @mul_v16i8c(<16 x i8> %i) nounwind { 9; SSE2-LABEL: mul_v16i8c: 10; SSE2: # %bb.0: # %entry 11; SSE2-NEXT: movdqa %xmm0, %xmm1 12; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] 13; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117] 14; SSE2-NEXT: pmullw %xmm2, %xmm1 15; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255] 16; SSE2-NEXT: pand %xmm3, %xmm1 17; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 18; SSE2-NEXT: pmullw %xmm2, %xmm0 19; SSE2-NEXT: pand %xmm3, %xmm0 20; SSE2-NEXT: packuswb %xmm1, %xmm0 21; SSE2-NEXT: retq 22; 23; SSE41-LABEL: mul_v16i8c: 24; SSE41: # %bb.0: # %entry 25; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 26; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] 27; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117] 28; SSE41-NEXT: pmullw %xmm2, %xmm0 29; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255] 30; SSE41-NEXT: pand %xmm3, %xmm0 31; SSE41-NEXT: pmullw %xmm2, %xmm1 32; SSE41-NEXT: pand %xmm3, %xmm1 33; SSE41-NEXT: packuswb %xmm0, %xmm1 34; SSE41-NEXT: movdqa %xmm1, %xmm0 35; SSE41-NEXT: retq 36; 37; AVX2-LABEL: mul_v16i8c: 38; AVX2: # %bb.0: # %entry 39; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero 40; AVX2-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm0 41; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 42; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 43; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0 44; AVX2-NEXT: vzeroupper 45; AVX2-NEXT: retq 46; 47; AVX512F-LABEL: mul_v16i8c: 48; AVX512F: # %bb.0: # %entry 49; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero 50; AVX512F-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm0 51; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero 52; AVX512F-NEXT: vpmovdb %zmm0, %xmm0 53; AVX512F-NEXT: vzeroupper 54; AVX512F-NEXT: retq 55; 56; AVX512BW-LABEL: mul_v16i8c: 57; AVX512BW: # %bb.0: # %entry 58; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero 59; AVX512BW-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm0 60; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0 61; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 62; AVX512BW-NEXT: vzeroupper 63; AVX512BW-NEXT: retq 64entry: 65 %A = mul <16 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 > 66 ret <16 x i8> %A 67} 68 69define <8 x i16> @mul_v8i16c(<8 x i16> %i) nounwind { 70; SSE-LABEL: mul_v8i16c: 71; SSE: # %bb.0: # %entry 72; SSE-NEXT: pmullw {{.*}}(%rip), %xmm0 73; SSE-NEXT: retq 74; 75; AVX-LABEL: mul_v8i16c: 76; AVX: # %bb.0: # %entry 77; AVX-NEXT: vpmullw {{.*}}(%rip), %xmm0, %xmm0 78; AVX-NEXT: retq 79entry: 80 %A = mul <8 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 > 81 ret <8 x i16> %A 82} 83 84define <4 x i32> @mul_v4i32c(<4 x i32> %i) nounwind { 85; SSE2-LABEL: mul_v4i32c: 86; SSE2: # %bb.0: # %entry 87; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [117,117,117,117] 88; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] 89; SSE2-NEXT: pmuludq %xmm1, %xmm0 90; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] 91; SSE2-NEXT: pmuludq %xmm1, %xmm2 92; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] 93; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] 94; SSE2-NEXT: retq 95; 96; SSE41-LABEL: mul_v4i32c: 97; SSE41: # %bb.0: # %entry 98; SSE41-NEXT: pmulld {{.*}}(%rip), %xmm0 99; SSE41-NEXT: retq 100; 101; AVX-LABEL: mul_v4i32c: 102; AVX: # %bb.0: # %entry 103; AVX-NEXT: vpbroadcastd {{.*#+}} xmm1 = [117,117,117,117] 104; AVX-NEXT: vpmulld %xmm1, %xmm0, %xmm0 105; AVX-NEXT: retq 106entry: 107 %A = mul <4 x i32> %i, < i32 117, i32 117, i32 117, i32 117 > 108 ret <4 x i32> %A 109} 110 111define <2 x i64> @mul_v2i64c(<2 x i64> %i) nounwind { 112; SSE-LABEL: mul_v2i64c: 113; SSE: # %bb.0: # %entry 114; SSE-NEXT: movdqa {{.*#+}} xmm1 = [117,117] 115; SSE-NEXT: movdqa %xmm0, %xmm2 116; SSE-NEXT: pmuludq %xmm1, %xmm2 117; SSE-NEXT: psrlq $32, %xmm0 118; SSE-NEXT: pmuludq %xmm1, %xmm0 119; SSE-NEXT: psllq $32, %xmm0 120; SSE-NEXT: paddq %xmm2, %xmm0 121; SSE-NEXT: retq 122; 123; AVX-LABEL: mul_v2i64c: 124; AVX: # %bb.0: # %entry 125; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [117,117] 126; AVX-NEXT: vpmuludq %xmm1, %xmm0, %xmm2 127; AVX-NEXT: vpsrlq $32, %xmm0, %xmm0 128; AVX-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 129; AVX-NEXT: vpsllq $32, %xmm0, %xmm0 130; AVX-NEXT: vpaddq %xmm0, %xmm2, %xmm0 131; AVX-NEXT: retq 132entry: 133 %A = mul <2 x i64> %i, < i64 117, i64 117 > 134 ret <2 x i64> %A 135} 136 137define <16 x i8> @mul_v16i8(<16 x i8> %i, <16 x i8> %j) nounwind { 138; SSE2-LABEL: mul_v16i8: 139; SSE2: # %bb.0: # %entry 140; SSE2-NEXT: movdqa %xmm1, %xmm2 141; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 142; SSE2-NEXT: movdqa %xmm0, %xmm3 143; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] 144; SSE2-NEXT: pmullw %xmm2, %xmm3 145; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255] 146; SSE2-NEXT: pand %xmm2, %xmm3 147; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 148; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 149; SSE2-NEXT: pmullw %xmm1, %xmm0 150; SSE2-NEXT: pand %xmm2, %xmm0 151; SSE2-NEXT: packuswb %xmm3, %xmm0 152; SSE2-NEXT: retq 153; 154; SSE41-LABEL: mul_v16i8: 155; SSE41: # %bb.0: # %entry 156; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 157; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] 158; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 159; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] 160; SSE41-NEXT: pmullw %xmm1, %xmm0 161; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] 162; SSE41-NEXT: pand %xmm1, %xmm0 163; SSE41-NEXT: pmullw %xmm3, %xmm2 164; SSE41-NEXT: pand %xmm1, %xmm2 165; SSE41-NEXT: packuswb %xmm0, %xmm2 166; SSE41-NEXT: movdqa %xmm2, %xmm0 167; SSE41-NEXT: retq 168; 169; AVX2-LABEL: mul_v16i8: 170; AVX2: # %bb.0: # %entry 171; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero 172; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero 173; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0 174; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 175; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 176; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0 177; AVX2-NEXT: vzeroupper 178; AVX2-NEXT: retq 179; 180; AVX512F-LABEL: mul_v16i8: 181; AVX512F: # %bb.0: # %entry 182; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero 183; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero 184; AVX512F-NEXT: vpmullw %ymm1, %ymm0, %ymm0 185; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero 186; AVX512F-NEXT: vpmovdb %zmm0, %xmm0 187; AVX512F-NEXT: vzeroupper 188; AVX512F-NEXT: retq 189; 190; AVX512BW-LABEL: mul_v16i8: 191; AVX512BW: # %bb.0: # %entry 192; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero 193; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero 194; AVX512BW-NEXT: vpmullw %ymm1, %ymm0, %ymm0 195; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0 196; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 197; AVX512BW-NEXT: vzeroupper 198; AVX512BW-NEXT: retq 199entry: 200 %A = mul <16 x i8> %i, %j 201 ret <16 x i8> %A 202} 203 204define <8 x i16> @mul_v8i16(<8 x i16> %i, <8 x i16> %j) nounwind { 205; SSE-LABEL: mul_v8i16: 206; SSE: # %bb.0: # %entry 207; SSE-NEXT: pmullw %xmm1, %xmm0 208; SSE-NEXT: retq 209; 210; AVX-LABEL: mul_v8i16: 211; AVX: # %bb.0: # %entry 212; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0 213; AVX-NEXT: retq 214entry: 215 %A = mul <8 x i16> %i, %j 216 ret <8 x i16> %A 217} 218 219define <4 x i32> @mul_v4i32(<4 x i32> %i, <4 x i32> %j) nounwind { 220; SSE2-LABEL: mul_v4i32: 221; SSE2: # %bb.0: # %entry 222; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] 223; SSE2-NEXT: pmuludq %xmm1, %xmm0 224; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] 225; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] 226; SSE2-NEXT: pmuludq %xmm2, %xmm1 227; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] 228; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] 229; SSE2-NEXT: retq 230; 231; SSE41-LABEL: mul_v4i32: 232; SSE41: # %bb.0: # %entry 233; SSE41-NEXT: pmulld %xmm1, %xmm0 234; SSE41-NEXT: retq 235; 236; AVX-LABEL: mul_v4i32: 237; AVX: # %bb.0: # %entry 238; AVX-NEXT: vpmulld %xmm1, %xmm0, %xmm0 239; AVX-NEXT: retq 240entry: 241 %A = mul <4 x i32> %i, %j 242 ret <4 x i32> %A 243} 244 245define <2 x i64> @mul_v2i64(<2 x i64> %i, <2 x i64> %j) nounwind { 246; SSE-LABEL: mul_v2i64: 247; SSE: # %bb.0: # %entry 248; SSE-NEXT: movdqa %xmm0, %xmm2 249; SSE-NEXT: psrlq $32, %xmm2 250; SSE-NEXT: pmuludq %xmm1, %xmm2 251; SSE-NEXT: movdqa %xmm1, %xmm3 252; SSE-NEXT: psrlq $32, %xmm3 253; SSE-NEXT: pmuludq %xmm0, %xmm3 254; SSE-NEXT: paddq %xmm2, %xmm3 255; SSE-NEXT: psllq $32, %xmm3 256; SSE-NEXT: pmuludq %xmm1, %xmm0 257; SSE-NEXT: paddq %xmm3, %xmm0 258; SSE-NEXT: retq 259; 260; AVX-LABEL: mul_v2i64: 261; AVX: # %bb.0: # %entry 262; AVX-NEXT: vpsrlq $32, %xmm0, %xmm2 263; AVX-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 264; AVX-NEXT: vpsrlq $32, %xmm1, %xmm3 265; AVX-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 266; AVX-NEXT: vpaddq %xmm2, %xmm3, %xmm2 267; AVX-NEXT: vpsllq $32, %xmm2, %xmm2 268; AVX-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 269; AVX-NEXT: vpaddq %xmm2, %xmm0, %xmm0 270; AVX-NEXT: retq 271entry: 272 %A = mul <2 x i64> %i, %j 273 ret <2 x i64> %A 274} 275 276declare void @foo() 277 278define <4 x i32> @mul_v4i32spill(<4 x i32> %i, <4 x i32> %j) nounwind { 279; SSE2-LABEL: mul_v4i32spill: 280; SSE2: # %bb.0: # %entry 281; SSE2-NEXT: subq $40, %rsp 282; SSE2-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 283; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill 284; SSE2-NEXT: callq foo 285; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload 286; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] 287; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload 288; SSE2-NEXT: pmuludq %xmm2, %xmm0 289; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] 290; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] 291; SSE2-NEXT: pmuludq %xmm1, %xmm2 292; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] 293; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] 294; SSE2-NEXT: addq $40, %rsp 295; SSE2-NEXT: retq 296; 297; SSE41-LABEL: mul_v4i32spill: 298; SSE41: # %bb.0: # %entry 299; SSE41-NEXT: subq $40, %rsp 300; SSE41-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 301; SSE41-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill 302; SSE41-NEXT: callq foo 303; SSE41-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload 304; SSE41-NEXT: pmulld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload 305; SSE41-NEXT: addq $40, %rsp 306; SSE41-NEXT: retq 307; 308; AVX-LABEL: mul_v4i32spill: 309; AVX: # %bb.0: # %entry 310; AVX-NEXT: subq $40, %rsp 311; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 312; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill 313; AVX-NEXT: callq foo 314; AVX-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload 315; AVX-NEXT: vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload 316; AVX-NEXT: addq $40, %rsp 317; AVX-NEXT: retq 318entry: 319 ; Use a call to force spills. 320 call void @foo() 321 %A = mul <4 x i32> %i, %j 322 ret <4 x i32> %A 323} 324 325define <2 x i64> @mul_v2i64spill(<2 x i64> %i, <2 x i64> %j) nounwind { 326; SSE-LABEL: mul_v2i64spill: 327; SSE: # %bb.0: # %entry 328; SSE-NEXT: subq $40, %rsp 329; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 330; SSE-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill 331; SSE-NEXT: callq foo 332; SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload 333; SSE-NEXT: movdqa %xmm0, %xmm2 334; SSE-NEXT: psrlq $32, %xmm2 335; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload 336; SSE-NEXT: pmuludq %xmm3, %xmm2 337; SSE-NEXT: movdqa %xmm3, %xmm1 338; SSE-NEXT: psrlq $32, %xmm1 339; SSE-NEXT: pmuludq %xmm0, %xmm1 340; SSE-NEXT: paddq %xmm2, %xmm1 341; SSE-NEXT: psllq $32, %xmm1 342; SSE-NEXT: pmuludq %xmm3, %xmm0 343; SSE-NEXT: paddq %xmm1, %xmm0 344; SSE-NEXT: addq $40, %rsp 345; SSE-NEXT: retq 346; 347; AVX-LABEL: mul_v2i64spill: 348; AVX: # %bb.0: # %entry 349; AVX-NEXT: subq $40, %rsp 350; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 351; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill 352; AVX-NEXT: callq foo 353; AVX-NEXT: vmovdqa (%rsp), %xmm3 # 16-byte Reload 354; AVX-NEXT: vpsrlq $32, %xmm3, %xmm0 355; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload 356; AVX-NEXT: vpmuludq %xmm2, %xmm0, %xmm0 357; AVX-NEXT: vpsrlq $32, %xmm2, %xmm1 358; AVX-NEXT: vpmuludq %xmm1, %xmm3, %xmm1 359; AVX-NEXT: vpaddq %xmm0, %xmm1, %xmm0 360; AVX-NEXT: vpsllq $32, %xmm0, %xmm0 361; AVX-NEXT: vpmuludq %xmm2, %xmm3, %xmm1 362; AVX-NEXT: vpaddq %xmm0, %xmm1, %xmm0 363; AVX-NEXT: addq $40, %rsp 364; AVX-NEXT: retq 365entry: 366 ; Use a call to force spills. 367 call void @foo() 368 %A = mul <2 x i64> %i, %j 369 ret <2 x i64> %A 370} 371 372define <32 x i8> @mul_v32i8c(<32 x i8> %i) nounwind { 373; SSE2-LABEL: mul_v32i8c: 374; SSE2: # %bb.0: # %entry 375; SSE2-NEXT: movdqa %xmm0, %xmm2 376; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 377; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [117,117,117,117,117,117,117,117] 378; SSE2-NEXT: pmullw %xmm3, %xmm2 379; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255] 380; SSE2-NEXT: pand %xmm4, %xmm2 381; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 382; SSE2-NEXT: pmullw %xmm3, %xmm0 383; SSE2-NEXT: pand %xmm4, %xmm0 384; SSE2-NEXT: packuswb %xmm2, %xmm0 385; SSE2-NEXT: movdqa %xmm1, %xmm2 386; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 387; SSE2-NEXT: pmullw %xmm3, %xmm2 388; SSE2-NEXT: pand %xmm4, %xmm2 389; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 390; SSE2-NEXT: pmullw %xmm3, %xmm1 391; SSE2-NEXT: pand %xmm4, %xmm1 392; SSE2-NEXT: packuswb %xmm2, %xmm1 393; SSE2-NEXT: retq 394; 395; SSE41-LABEL: mul_v32i8c: 396; SSE41: # %bb.0: # %entry 397; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 398; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] 399; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [117,117,117,117,117,117,117,117] 400; SSE41-NEXT: pmullw %xmm4, %xmm0 401; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255] 402; SSE41-NEXT: pand %xmm5, %xmm0 403; SSE41-NEXT: pmullw %xmm4, %xmm2 404; SSE41-NEXT: pand %xmm5, %xmm2 405; SSE41-NEXT: packuswb %xmm0, %xmm2 406; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 407; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] 408; SSE41-NEXT: pmullw %xmm4, %xmm1 409; SSE41-NEXT: pand %xmm5, %xmm1 410; SSE41-NEXT: pmullw %xmm4, %xmm3 411; SSE41-NEXT: pand %xmm5, %xmm3 412; SSE41-NEXT: packuswb %xmm1, %xmm3 413; SSE41-NEXT: movdqa %xmm2, %xmm0 414; SSE41-NEXT: movdqa %xmm3, %xmm1 415; SSE41-NEXT: retq 416; 417; AVX2-LABEL: mul_v32i8c: 418; AVX2: # %bb.0: # %entry 419; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 420; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117] 421; AVX2-NEXT: vpmullw %ymm2, %ymm1, %ymm1 422; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 423; AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1 424; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 425; AVX2-NEXT: vpmullw %ymm2, %ymm0, %ymm0 426; AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0 427; AVX2-NEXT: vpackuswb %ymm1, %ymm0, %ymm0 428; AVX2-NEXT: retq 429; 430; AVX512F-LABEL: mul_v32i8c: 431; AVX512F: # %bb.0: # %entry 432; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 433; AVX512F-NEXT: vmovdqa {{.*#+}} ymm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117] 434; AVX512F-NEXT: vpmullw %ymm2, %ymm1, %ymm1 435; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 436; AVX512F-NEXT: vpand %ymm3, %ymm1, %ymm1 437; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 438; AVX512F-NEXT: vpmullw %ymm2, %ymm0, %ymm0 439; AVX512F-NEXT: vpand %ymm3, %ymm0, %ymm0 440; AVX512F-NEXT: vpackuswb %ymm1, %ymm0, %ymm0 441; AVX512F-NEXT: retq 442; 443; AVX512BW-LABEL: mul_v32i8c: 444; AVX512BW: # %bb.0: # %entry 445; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero 446; AVX512BW-NEXT: vpmullw {{.*}}(%rip), %zmm0, %zmm0 447; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0 448; AVX512BW-NEXT: retq 449entry: 450 %A = mul <32 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 > 451 ret <32 x i8> %A 452} 453 454define <16 x i16> @mul_v16i16c(<16 x i16> %i) nounwind { 455; SSE-LABEL: mul_v16i16c: 456; SSE: # %bb.0: # %entry 457; SSE-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117] 458; SSE-NEXT: pmullw %xmm2, %xmm0 459; SSE-NEXT: pmullw %xmm2, %xmm1 460; SSE-NEXT: retq 461; 462; AVX-LABEL: mul_v16i16c: 463; AVX: # %bb.0: # %entry 464; AVX-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm0 465; AVX-NEXT: retq 466entry: 467 %A = mul <16 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 > 468 ret <16 x i16> %A 469} 470 471define <8 x i32> @mul_v8i32c(<8 x i32> %i) nounwind { 472; SSE2-LABEL: mul_v8i32c: 473; SSE2: # %bb.0: # %entry 474; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117] 475; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] 476; SSE2-NEXT: pmuludq %xmm2, %xmm0 477; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] 478; SSE2-NEXT: pmuludq %xmm2, %xmm3 479; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] 480; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] 481; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3] 482; SSE2-NEXT: pmuludq %xmm2, %xmm1 483; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] 484; SSE2-NEXT: pmuludq %xmm2, %xmm3 485; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3] 486; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] 487; SSE2-NEXT: retq 488; 489; SSE41-LABEL: mul_v8i32c: 490; SSE41: # %bb.0: # %entry 491; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117] 492; SSE41-NEXT: pmulld %xmm2, %xmm0 493; SSE41-NEXT: pmulld %xmm2, %xmm1 494; SSE41-NEXT: retq 495; 496; AVX-LABEL: mul_v8i32c: 497; AVX: # %bb.0: # %entry 498; AVX-NEXT: vpbroadcastd {{.*#+}} ymm1 = [117,117,117,117,117,117,117,117] 499; AVX-NEXT: vpmulld %ymm1, %ymm0, %ymm0 500; AVX-NEXT: retq 501entry: 502 %A = mul <8 x i32> %i, < i32 117, i32 117, i32 117, i32 117, i32 117, i32 117, i32 117, i32 117 > 503 ret <8 x i32> %A 504} 505 506define <4 x i64> @mul_v4i64c(<4 x i64> %i) nounwind { 507; SSE-LABEL: mul_v4i64c: 508; SSE: # %bb.0: # %entry 509; SSE-NEXT: movdqa {{.*#+}} xmm2 = [117,117] 510; SSE-NEXT: movdqa %xmm0, %xmm3 511; SSE-NEXT: pmuludq %xmm2, %xmm3 512; SSE-NEXT: psrlq $32, %xmm0 513; SSE-NEXT: pmuludq %xmm2, %xmm0 514; SSE-NEXT: psllq $32, %xmm0 515; SSE-NEXT: paddq %xmm3, %xmm0 516; SSE-NEXT: movdqa %xmm1, %xmm3 517; SSE-NEXT: pmuludq %xmm2, %xmm3 518; SSE-NEXT: psrlq $32, %xmm1 519; SSE-NEXT: pmuludq %xmm2, %xmm1 520; SSE-NEXT: psllq $32, %xmm1 521; SSE-NEXT: paddq %xmm3, %xmm1 522; SSE-NEXT: retq 523; 524; AVX-LABEL: mul_v4i64c: 525; AVX: # %bb.0: # %entry 526; AVX-NEXT: vpbroadcastq {{.*#+}} ymm1 = [117,117,117,117] 527; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm2 528; AVX-NEXT: vpsrlq $32, %ymm0, %ymm0 529; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 530; AVX-NEXT: vpsllq $32, %ymm0, %ymm0 531; AVX-NEXT: vpaddq %ymm0, %ymm2, %ymm0 532; AVX-NEXT: retq 533entry: 534 %A = mul <4 x i64> %i, < i64 117, i64 117, i64 117, i64 117 > 535 ret <4 x i64> %A 536} 537 538define <32 x i8> @mul_v32i8(<32 x i8> %i, <32 x i8> %j) nounwind { 539; SSE2-LABEL: mul_v32i8: 540; SSE2: # %bb.0: # %entry 541; SSE2-NEXT: movdqa %xmm2, %xmm4 542; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] 543; SSE2-NEXT: movdqa %xmm0, %xmm5 544; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15] 545; SSE2-NEXT: pmullw %xmm4, %xmm5 546; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255] 547; SSE2-NEXT: pand %xmm4, %xmm5 548; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] 549; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 550; SSE2-NEXT: pmullw %xmm2, %xmm0 551; SSE2-NEXT: pand %xmm4, %xmm0 552; SSE2-NEXT: packuswb %xmm5, %xmm0 553; SSE2-NEXT: movdqa %xmm3, %xmm2 554; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 555; SSE2-NEXT: movdqa %xmm1, %xmm5 556; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15] 557; SSE2-NEXT: pmullw %xmm2, %xmm5 558; SSE2-NEXT: pand %xmm4, %xmm5 559; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 560; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 561; SSE2-NEXT: pmullw %xmm3, %xmm1 562; SSE2-NEXT: pand %xmm4, %xmm1 563; SSE2-NEXT: packuswb %xmm5, %xmm1 564; SSE2-NEXT: retq 565; 566; SSE41-LABEL: mul_v32i8: 567; SSE41: # %bb.0: # %entry 568; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero 569; SSE41-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 570; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 571; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] 572; SSE41-NEXT: pmullw %xmm2, %xmm0 573; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255] 574; SSE41-NEXT: pand %xmm6, %xmm0 575; SSE41-NEXT: pmullw %xmm5, %xmm4 576; SSE41-NEXT: pand %xmm6, %xmm4 577; SSE41-NEXT: packuswb %xmm0, %xmm4 578; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero 579; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] 580; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 581; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] 582; SSE41-NEXT: pmullw %xmm3, %xmm1 583; SSE41-NEXT: pand %xmm6, %xmm1 584; SSE41-NEXT: pmullw %xmm0, %xmm2 585; SSE41-NEXT: pand %xmm6, %xmm2 586; SSE41-NEXT: packuswb %xmm1, %xmm2 587; SSE41-NEXT: movdqa %xmm4, %xmm0 588; SSE41-NEXT: movdqa %xmm2, %xmm1 589; SSE41-NEXT: retq 590; 591; AVX2-LABEL: mul_v32i8: 592; AVX2: # %bb.0: # %entry 593; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31] 594; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 595; AVX2-NEXT: vpmullw %ymm2, %ymm3, %ymm2 596; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 597; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2 598; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23] 599; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 600; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0 601; AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0 602; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 603; AVX2-NEXT: retq 604; 605; AVX512F-LABEL: mul_v32i8: 606; AVX512F: # %bb.0: # %entry 607; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31] 608; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 609; AVX512F-NEXT: vpmullw %ymm2, %ymm3, %ymm2 610; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 611; AVX512F-NEXT: vpand %ymm3, %ymm2, %ymm2 612; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23] 613; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 614; AVX512F-NEXT: vpmullw %ymm1, %ymm0, %ymm0 615; AVX512F-NEXT: vpand %ymm3, %ymm0, %ymm0 616; AVX512F-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 617; AVX512F-NEXT: retq 618; 619; AVX512BW-LABEL: mul_v32i8: 620; AVX512BW: # %bb.0: # %entry 621; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero 622; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero 623; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0 624; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0 625; AVX512BW-NEXT: retq 626entry: 627 %A = mul <32 x i8> %i, %j 628 ret <32 x i8> %A 629} 630 631define <16 x i16> @mul_v16i16(<16 x i16> %i, <16 x i16> %j) nounwind { 632; SSE-LABEL: mul_v16i16: 633; SSE: # %bb.0: # %entry 634; SSE-NEXT: pmullw %xmm2, %xmm0 635; SSE-NEXT: pmullw %xmm3, %xmm1 636; SSE-NEXT: retq 637; 638; AVX-LABEL: mul_v16i16: 639; AVX: # %bb.0: # %entry 640; AVX-NEXT: vpmullw %ymm1, %ymm0, %ymm0 641; AVX-NEXT: retq 642entry: 643 %A = mul <16 x i16> %i, %j 644 ret <16 x i16> %A 645} 646 647define <8 x i32> @mul_v8i32(<8 x i32> %i, <8 x i32> %j) nounwind { 648; SSE2-LABEL: mul_v8i32: 649; SSE2: # %bb.0: # %entry 650; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] 651; SSE2-NEXT: pmuludq %xmm2, %xmm0 652; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] 653; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] 654; SSE2-NEXT: pmuludq %xmm4, %xmm2 655; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] 656; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] 657; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3] 658; SSE2-NEXT: pmuludq %xmm3, %xmm1 659; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] 660; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3] 661; SSE2-NEXT: pmuludq %xmm2, %xmm3 662; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3] 663; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] 664; SSE2-NEXT: retq 665; 666; SSE41-LABEL: mul_v8i32: 667; SSE41: # %bb.0: # %entry 668; SSE41-NEXT: pmulld %xmm2, %xmm0 669; SSE41-NEXT: pmulld %xmm3, %xmm1 670; SSE41-NEXT: retq 671; 672; AVX-LABEL: mul_v8i32: 673; AVX: # %bb.0: # %entry 674; AVX-NEXT: vpmulld %ymm1, %ymm0, %ymm0 675; AVX-NEXT: retq 676entry: 677 %A = mul <8 x i32> %i, %j 678 ret <8 x i32> %A 679} 680 681define <4 x i64> @mul_v4i64(<4 x i64> %i, <4 x i64> %j) nounwind { 682; SSE-LABEL: mul_v4i64: 683; SSE: # %bb.0: # %entry 684; SSE-NEXT: movdqa %xmm0, %xmm4 685; SSE-NEXT: psrlq $32, %xmm4 686; SSE-NEXT: pmuludq %xmm2, %xmm4 687; SSE-NEXT: movdqa %xmm2, %xmm5 688; SSE-NEXT: psrlq $32, %xmm5 689; SSE-NEXT: pmuludq %xmm0, %xmm5 690; SSE-NEXT: paddq %xmm4, %xmm5 691; SSE-NEXT: psllq $32, %xmm5 692; SSE-NEXT: pmuludq %xmm2, %xmm0 693; SSE-NEXT: paddq %xmm5, %xmm0 694; SSE-NEXT: movdqa %xmm1, %xmm2 695; SSE-NEXT: psrlq $32, %xmm2 696; SSE-NEXT: pmuludq %xmm3, %xmm2 697; SSE-NEXT: movdqa %xmm3, %xmm4 698; SSE-NEXT: psrlq $32, %xmm4 699; SSE-NEXT: pmuludq %xmm1, %xmm4 700; SSE-NEXT: paddq %xmm2, %xmm4 701; SSE-NEXT: psllq $32, %xmm4 702; SSE-NEXT: pmuludq %xmm3, %xmm1 703; SSE-NEXT: paddq %xmm4, %xmm1 704; SSE-NEXT: retq 705; 706; AVX-LABEL: mul_v4i64: 707; AVX: # %bb.0: # %entry 708; AVX-NEXT: vpsrlq $32, %ymm0, %ymm2 709; AVX-NEXT: vpmuludq %ymm1, %ymm2, %ymm2 710; AVX-NEXT: vpsrlq $32, %ymm1, %ymm3 711; AVX-NEXT: vpmuludq %ymm3, %ymm0, %ymm3 712; AVX-NEXT: vpaddq %ymm2, %ymm3, %ymm2 713; AVX-NEXT: vpsllq $32, %ymm2, %ymm2 714; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 715; AVX-NEXT: vpaddq %ymm2, %ymm0, %ymm0 716; AVX-NEXT: retq 717entry: 718 %A = mul <4 x i64> %i, %j 719 ret <4 x i64> %A 720} 721 722define <64 x i8> @mul_v64i8c(<64 x i8> %i) nounwind { 723; SSE2-LABEL: mul_v64i8c: 724; SSE2: # %bb.0: # %entry 725; SSE2-NEXT: movdqa %xmm0, %xmm6 726; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15] 727; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [117,117,117,117,117,117,117,117] 728; SSE2-NEXT: pmullw %xmm4, %xmm6 729; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255] 730; SSE2-NEXT: pand %xmm5, %xmm6 731; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 732; SSE2-NEXT: pmullw %xmm4, %xmm0 733; SSE2-NEXT: pand %xmm5, %xmm0 734; SSE2-NEXT: packuswb %xmm6, %xmm0 735; SSE2-NEXT: movdqa %xmm1, %xmm6 736; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15] 737; SSE2-NEXT: pmullw %xmm4, %xmm6 738; SSE2-NEXT: pand %xmm5, %xmm6 739; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 740; SSE2-NEXT: pmullw %xmm4, %xmm1 741; SSE2-NEXT: pand %xmm5, %xmm1 742; SSE2-NEXT: packuswb %xmm6, %xmm1 743; SSE2-NEXT: movdqa %xmm2, %xmm6 744; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15] 745; SSE2-NEXT: pmullw %xmm4, %xmm6 746; SSE2-NEXT: pand %xmm5, %xmm6 747; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] 748; SSE2-NEXT: pmullw %xmm4, %xmm2 749; SSE2-NEXT: pand %xmm5, %xmm2 750; SSE2-NEXT: packuswb %xmm6, %xmm2 751; SSE2-NEXT: movdqa %xmm3, %xmm6 752; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15] 753; SSE2-NEXT: pmullw %xmm4, %xmm6 754; SSE2-NEXT: pand %xmm5, %xmm6 755; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 756; SSE2-NEXT: pmullw %xmm4, %xmm3 757; SSE2-NEXT: pand %xmm5, %xmm3 758; SSE2-NEXT: packuswb %xmm6, %xmm3 759; SSE2-NEXT: retq 760; 761; SSE41-LABEL: mul_v64i8c: 762; SSE41: # %bb.0: # %entry 763; SSE41-NEXT: movdqa %xmm1, %xmm4 764; SSE41-NEXT: movdqa %xmm0, %xmm1 765; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 766; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] 767; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [117,117,117,117,117,117,117,117] 768; SSE41-NEXT: pmullw %xmm6, %xmm1 769; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255] 770; SSE41-NEXT: pand %xmm7, %xmm1 771; SSE41-NEXT: pmullw %xmm6, %xmm0 772; SSE41-NEXT: pand %xmm7, %xmm0 773; SSE41-NEXT: packuswb %xmm1, %xmm0 774; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero 775; SSE41-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] 776; SSE41-NEXT: pmullw %xmm6, %xmm4 777; SSE41-NEXT: pand %xmm7, %xmm4 778; SSE41-NEXT: pmullw %xmm6, %xmm1 779; SSE41-NEXT: pand %xmm7, %xmm1 780; SSE41-NEXT: packuswb %xmm4, %xmm1 781; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero 782; SSE41-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 783; SSE41-NEXT: pmullw %xmm6, %xmm2 784; SSE41-NEXT: pand %xmm7, %xmm2 785; SSE41-NEXT: pmullw %xmm6, %xmm4 786; SSE41-NEXT: pand %xmm7, %xmm4 787; SSE41-NEXT: packuswb %xmm2, %xmm4 788; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero 789; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] 790; SSE41-NEXT: pmullw %xmm6, %xmm3 791; SSE41-NEXT: pand %xmm7, %xmm3 792; SSE41-NEXT: pmullw %xmm6, %xmm5 793; SSE41-NEXT: pand %xmm7, %xmm5 794; SSE41-NEXT: packuswb %xmm3, %xmm5 795; SSE41-NEXT: movdqa %xmm4, %xmm2 796; SSE41-NEXT: movdqa %xmm5, %xmm3 797; SSE41-NEXT: retq 798; 799; AVX2-LABEL: mul_v64i8c: 800; AVX2: # %bb.0: # %entry 801; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 802; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117] 803; AVX2-NEXT: vpmullw %ymm3, %ymm2, %ymm2 804; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 805; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2 806; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 807; AVX2-NEXT: vpmullw %ymm3, %ymm0, %ymm0 808; AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0 809; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 810; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31] 811; AVX2-NEXT: vpmullw %ymm3, %ymm2, %ymm2 812; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2 813; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23] 814; AVX2-NEXT: vpmullw %ymm3, %ymm1, %ymm1 815; AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1 816; AVX2-NEXT: vpackuswb %ymm2, %ymm1, %ymm1 817; AVX2-NEXT: retq 818; 819; AVX512F-LABEL: mul_v64i8c: 820; AVX512F: # %bb.0: # %entry 821; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 822; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117] 823; AVX512F-NEXT: vpmullw %ymm3, %ymm2, %ymm2 824; AVX512F-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 825; AVX512F-NEXT: vpand %ymm4, %ymm2, %ymm2 826; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 827; AVX512F-NEXT: vpmullw %ymm3, %ymm0, %ymm0 828; AVX512F-NEXT: vpand %ymm4, %ymm0, %ymm0 829; AVX512F-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 830; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31] 831; AVX512F-NEXT: vpmullw %ymm3, %ymm2, %ymm2 832; AVX512F-NEXT: vpand %ymm4, %ymm2, %ymm2 833; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23] 834; AVX512F-NEXT: vpmullw %ymm3, %ymm1, %ymm1 835; AVX512F-NEXT: vpand %ymm4, %ymm1, %ymm1 836; AVX512F-NEXT: vpackuswb %ymm2, %ymm1, %ymm1 837; AVX512F-NEXT: retq 838; 839; AVX512BW-LABEL: mul_v64i8c: 840; AVX512BW: # %bb.0: # %entry 841; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] 842; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117] 843; AVX512BW-NEXT: vpmullw %zmm2, %zmm1, %zmm1 844; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 845; AVX512BW-NEXT: vpandq %zmm3, %zmm1, %zmm1 846; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55] 847; AVX512BW-NEXT: vpmullw %zmm2, %zmm0, %zmm0 848; AVX512BW-NEXT: vpandq %zmm3, %zmm0, %zmm0 849; AVX512BW-NEXT: vpackuswb %zmm1, %zmm0, %zmm0 850; AVX512BW-NEXT: retq 851entry: 852 %A = mul <64 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 > 853 ret <64 x i8> %A 854} 855 856define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind { 857; SSE2-LABEL: mul_v64i8: 858; SSE2: # %bb.0: # %entry 859; SSE2-NEXT: movdqa %xmm4, %xmm8 860; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15] 861; SSE2-NEXT: movdqa %xmm0, %xmm9 862; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm0[8],xmm9[9],xmm0[9],xmm9[10],xmm0[10],xmm9[11],xmm0[11],xmm9[12],xmm0[12],xmm9[13],xmm0[13],xmm9[14],xmm0[14],xmm9[15],xmm0[15] 863; SSE2-NEXT: pmullw %xmm8, %xmm9 864; SSE2-NEXT: movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255] 865; SSE2-NEXT: pand %xmm8, %xmm9 866; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 867; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 868; SSE2-NEXT: pmullw %xmm4, %xmm0 869; SSE2-NEXT: pand %xmm8, %xmm0 870; SSE2-NEXT: packuswb %xmm9, %xmm0 871; SSE2-NEXT: movdqa %xmm5, %xmm9 872; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm0[8],xmm9[9],xmm0[9],xmm9[10],xmm0[10],xmm9[11],xmm0[11],xmm9[12],xmm0[12],xmm9[13],xmm0[13],xmm9[14],xmm0[14],xmm9[15],xmm0[15] 873; SSE2-NEXT: movdqa %xmm1, %xmm4 874; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] 875; SSE2-NEXT: pmullw %xmm9, %xmm4 876; SSE2-NEXT: pand %xmm8, %xmm4 877; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7] 878; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 879; SSE2-NEXT: pmullw %xmm5, %xmm1 880; SSE2-NEXT: pand %xmm8, %xmm1 881; SSE2-NEXT: packuswb %xmm4, %xmm1 882; SSE2-NEXT: movdqa %xmm6, %xmm4 883; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] 884; SSE2-NEXT: movdqa %xmm2, %xmm5 885; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15] 886; SSE2-NEXT: pmullw %xmm4, %xmm5 887; SSE2-NEXT: pand %xmm8, %xmm5 888; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7] 889; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] 890; SSE2-NEXT: pmullw %xmm6, %xmm2 891; SSE2-NEXT: pand %xmm8, %xmm2 892; SSE2-NEXT: packuswb %xmm5, %xmm2 893; SSE2-NEXT: movdqa %xmm7, %xmm4 894; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] 895; SSE2-NEXT: movdqa %xmm3, %xmm5 896; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15] 897; SSE2-NEXT: pmullw %xmm4, %xmm5 898; SSE2-NEXT: pand %xmm8, %xmm5 899; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7] 900; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 901; SSE2-NEXT: pmullw %xmm7, %xmm3 902; SSE2-NEXT: pand %xmm8, %xmm3 903; SSE2-NEXT: packuswb %xmm5, %xmm3 904; SSE2-NEXT: retq 905; 906; SSE41-LABEL: mul_v64i8: 907; SSE41: # %bb.0: # %entry 908; SSE41-NEXT: movdqa %xmm1, %xmm8 909; SSE41-NEXT: movdqa %xmm0, %xmm1 910; SSE41-NEXT: pmovzxbw {{.*#+}} xmm10 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero 911; SSE41-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] 912; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 913; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] 914; SSE41-NEXT: pmullw %xmm4, %xmm1 915; SSE41-NEXT: movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255] 916; SSE41-NEXT: pand %xmm9, %xmm1 917; SSE41-NEXT: pmullw %xmm10, %xmm0 918; SSE41-NEXT: pand %xmm9, %xmm0 919; SSE41-NEXT: packuswb %xmm1, %xmm0 920; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero 921; SSE41-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15] 922; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero,xmm8[4],zero,xmm8[5],zero,xmm8[6],zero,xmm8[7],zero 923; SSE41-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15] 924; SSE41-NEXT: pmullw %xmm5, %xmm8 925; SSE41-NEXT: pand %xmm9, %xmm8 926; SSE41-NEXT: pmullw %xmm4, %xmm1 927; SSE41-NEXT: pand %xmm9, %xmm1 928; SSE41-NEXT: packuswb %xmm8, %xmm1 929; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero 930; SSE41-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15] 931; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero 932; SSE41-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 933; SSE41-NEXT: pmullw %xmm6, %xmm2 934; SSE41-NEXT: pand %xmm9, %xmm2 935; SSE41-NEXT: pmullw %xmm5, %xmm4 936; SSE41-NEXT: pand %xmm9, %xmm4 937; SSE41-NEXT: packuswb %xmm2, %xmm4 938; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero 939; SSE41-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15] 940; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero 941; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] 942; SSE41-NEXT: pmullw %xmm7, %xmm3 943; SSE41-NEXT: pand %xmm9, %xmm3 944; SSE41-NEXT: pmullw %xmm2, %xmm5 945; SSE41-NEXT: pand %xmm9, %xmm5 946; SSE41-NEXT: packuswb %xmm3, %xmm5 947; SSE41-NEXT: movdqa %xmm4, %xmm2 948; SSE41-NEXT: movdqa %xmm5, %xmm3 949; SSE41-NEXT: retq 950; 951; AVX2-LABEL: mul_v64i8: 952; AVX2: # %bb.0: # %entry 953; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31] 954; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 955; AVX2-NEXT: vpmullw %ymm4, %ymm5, %ymm4 956; AVX2-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 957; AVX2-NEXT: vpand %ymm5, %ymm4, %ymm4 958; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23] 959; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 960; AVX2-NEXT: vpmullw %ymm2, %ymm0, %ymm0 961; AVX2-NEXT: vpand %ymm5, %ymm0, %ymm0 962; AVX2-NEXT: vpackuswb %ymm4, %ymm0, %ymm0 963; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31] 964; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31] 965; AVX2-NEXT: vpmullw %ymm2, %ymm4, %ymm2 966; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2 967; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23] 968; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23] 969; AVX2-NEXT: vpmullw %ymm3, %ymm1, %ymm1 970; AVX2-NEXT: vpand %ymm5, %ymm1, %ymm1 971; AVX2-NEXT: vpackuswb %ymm2, %ymm1, %ymm1 972; AVX2-NEXT: retq 973; 974; AVX512F-LABEL: mul_v64i8: 975; AVX512F: # %bb.0: # %entry 976; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31] 977; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 978; AVX512F-NEXT: vpmullw %ymm4, %ymm5, %ymm4 979; AVX512F-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 980; AVX512F-NEXT: vpand %ymm5, %ymm4, %ymm4 981; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23] 982; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 983; AVX512F-NEXT: vpmullw %ymm2, %ymm0, %ymm0 984; AVX512F-NEXT: vpand %ymm5, %ymm0, %ymm0 985; AVX512F-NEXT: vpackuswb %ymm4, %ymm0, %ymm0 986; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31] 987; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31] 988; AVX512F-NEXT: vpmullw %ymm2, %ymm4, %ymm2 989; AVX512F-NEXT: vpand %ymm5, %ymm2, %ymm2 990; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23] 991; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23] 992; AVX512F-NEXT: vpmullw %ymm3, %ymm1, %ymm1 993; AVX512F-NEXT: vpand %ymm5, %ymm1, %ymm1 994; AVX512F-NEXT: vpackuswb %ymm2, %ymm1, %ymm1 995; AVX512F-NEXT: retq 996; 997; AVX512BW-LABEL: mul_v64i8: 998; AVX512BW: # %bb.0: # %entry 999; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63] 1000; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] 1001; AVX512BW-NEXT: vpmullw %zmm2, %zmm3, %zmm2 1002; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 1003; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm2 1004; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55] 1005; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55] 1006; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0 1007; AVX512BW-NEXT: vpandq %zmm3, %zmm0, %zmm0 1008; AVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0 1009; AVX512BW-NEXT: retq 1010entry: 1011 %A = mul <64 x i8> %i, %j 1012 ret <64 x i8> %A 1013} 1014 1015; PR30845 1016define <4 x i32> @mul_v4i64_zero_upper(<4 x i32> %val1, <4 x i32> %val2) { 1017; SSE2-LABEL: mul_v4i64_zero_upper: 1018; SSE2: # %bb.0: # %entry 1019; SSE2-NEXT: pxor %xmm3, %xmm3 1020; SSE2-NEXT: movdqa %xmm0, %xmm2 1021; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] 1022; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3] 1023; SSE2-NEXT: movdqa %xmm1, %xmm4 1024; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] 1025; SSE2-NEXT: pmuludq %xmm4, %xmm2 1026; SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3] 1027; SSE2-NEXT: pmuludq %xmm0, %xmm1 1028; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[1,3] 1029; SSE2-NEXT: movaps %xmm2, %xmm0 1030; SSE2-NEXT: retq 1031; 1032; SSE41-LABEL: mul_v4i64_zero_upper: 1033; SSE41: # %bb.0: # %entry 1034; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero 1035; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,1,3,3] 1036; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero 1037; SSE41-NEXT: pmuludq %xmm2, %xmm0 1038; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,3,3] 1039; SSE41-NEXT: pmuludq %xmm3, %xmm1 1040; SSE41-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3] 1041; SSE41-NEXT: retq 1042; 1043; AVX-LABEL: mul_v4i64_zero_upper: 1044; AVX: # %bb.0: # %entry 1045; AVX-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 1046; AVX-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 1047; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 1048; AVX-NEXT: vextracti128 $1, %ymm0, %xmm1 1049; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3] 1050; AVX-NEXT: vzeroupper 1051; AVX-NEXT: retq 1052entry: 1053 %val1a = zext <4 x i32> %val1 to <4 x i64> 1054 %val2a = zext <4 x i32> %val2 to <4 x i64> 1055 %res64 = mul <4 x i64> %val1a, %val2a 1056 %rescast = bitcast <4 x i64> %res64 to <8 x i32> 1057 %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 1058 ret <4 x i32> %res 1059} 1060 1061define <4 x i32> @mul_v4i64_zero_upper_left(<4 x i32> %val1, <4 x i64> %val2) { 1062; SSE2-LABEL: mul_v4i64_zero_upper_left: 1063; SSE2: # %bb.0: # %entry 1064; SSE2-NEXT: pxor %xmm3, %xmm3 1065; SSE2-NEXT: movdqa %xmm0, %xmm4 1066; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] 1067; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3] 1068; SSE2-NEXT: movdqa %xmm0, %xmm3 1069; SSE2-NEXT: pmuludq %xmm2, %xmm3 1070; SSE2-NEXT: psrlq $32, %xmm2 1071; SSE2-NEXT: pmuludq %xmm0, %xmm2 1072; SSE2-NEXT: psllq $32, %xmm2 1073; SSE2-NEXT: paddq %xmm3, %xmm2 1074; SSE2-NEXT: movdqa %xmm4, %xmm0 1075; SSE2-NEXT: pmuludq %xmm1, %xmm0 1076; SSE2-NEXT: psrlq $32, %xmm1 1077; SSE2-NEXT: pmuludq %xmm4, %xmm1 1078; SSE2-NEXT: psllq $32, %xmm1 1079; SSE2-NEXT: paddq %xmm1, %xmm0 1080; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3] 1081; SSE2-NEXT: retq 1082; 1083; SSE41-LABEL: mul_v4i64_zero_upper_left: 1084; SSE41: # %bb.0: # %entry 1085; SSE41-NEXT: pxor %xmm3, %xmm3 1086; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero 1087; SSE41-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3] 1088; SSE41-NEXT: movdqa %xmm0, %xmm3 1089; SSE41-NEXT: pmuludq %xmm2, %xmm3 1090; SSE41-NEXT: psrlq $32, %xmm2 1091; SSE41-NEXT: pmuludq %xmm0, %xmm2 1092; SSE41-NEXT: psllq $32, %xmm2 1093; SSE41-NEXT: paddq %xmm3, %xmm2 1094; SSE41-NEXT: movdqa %xmm4, %xmm0 1095; SSE41-NEXT: pmuludq %xmm1, %xmm0 1096; SSE41-NEXT: psrlq $32, %xmm1 1097; SSE41-NEXT: pmuludq %xmm4, %xmm1 1098; SSE41-NEXT: psllq $32, %xmm1 1099; SSE41-NEXT: paddq %xmm1, %xmm0 1100; SSE41-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3] 1101; SSE41-NEXT: retq 1102; 1103; AVX-LABEL: mul_v4i64_zero_upper_left: 1104; AVX: # %bb.0: # %entry 1105; AVX-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 1106; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm2 1107; AVX-NEXT: vpsrlq $32, %ymm1, %ymm1 1108; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 1109; AVX-NEXT: vpsllq $32, %ymm0, %ymm0 1110; AVX-NEXT: vpaddq %ymm0, %ymm2, %ymm0 1111; AVX-NEXT: vextracti128 $1, %ymm0, %xmm1 1112; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3] 1113; AVX-NEXT: vzeroupper 1114; AVX-NEXT: retq 1115entry: 1116 %val1a = zext <4 x i32> %val1 to <4 x i64> 1117 %res64 = mul <4 x i64> %val1a, %val2 1118 %rescast = bitcast <4 x i64> %res64 to <8 x i32> 1119 %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 1120 ret <4 x i32> %res 1121} 1122 1123define <4 x i32> @mul_v4i64_zero_lower(<4 x i32> %val1, <4 x i64> %val2) { 1124; SSE2-LABEL: mul_v4i64_zero_lower: 1125; SSE2: # %bb.0: # %entry 1126; SSE2-NEXT: pxor %xmm4, %xmm4 1127; SSE2-NEXT: movdqa %xmm0, %xmm3 1128; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1] 1129; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm4[2],xmm0[3],xmm4[3] 1130; SSE2-NEXT: psrlq $32, %xmm2 1131; SSE2-NEXT: pmuludq %xmm0, %xmm2 1132; SSE2-NEXT: psrlq $32, %xmm1 1133; SSE2-NEXT: pmuludq %xmm1, %xmm3 1134; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm2[0,2] 1135; SSE2-NEXT: movaps %xmm3, %xmm0 1136; SSE2-NEXT: retq 1137; 1138; SSE41-LABEL: mul_v4i64_zero_lower: 1139; SSE41: # %bb.0: # %entry 1140; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero 1141; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,3,3] 1142; SSE41-NEXT: psrlq $32, %xmm2 1143; SSE41-NEXT: pmuludq %xmm0, %xmm2 1144; SSE41-NEXT: psrlq $32, %xmm1 1145; SSE41-NEXT: pmuludq %xmm1, %xmm3 1146; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm2[0,2] 1147; SSE41-NEXT: movaps %xmm3, %xmm0 1148; SSE41-NEXT: retq 1149; 1150; AVX-LABEL: mul_v4i64_zero_lower: 1151; AVX: # %bb.0: # %entry 1152; AVX-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 1153; AVX-NEXT: vpsrlq $32, %ymm1, %ymm1 1154; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 1155; AVX-NEXT: vpsllq $32, %ymm0, %ymm0 1156; AVX-NEXT: vextracti128 $1, %ymm0, %xmm1 1157; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3] 1158; AVX-NEXT: vzeroupper 1159; AVX-NEXT: retq 1160entry: 1161 %val1a = zext <4 x i32> %val1 to <4 x i64> 1162 %val2a = and <4 x i64> %val2, <i64 -4294967296, i64 -4294967296, i64 -4294967296, i64 -4294967296> 1163 %res64 = mul <4 x i64> %val1a, %val2a 1164 %rescast = bitcast <4 x i64> %res64 to <8 x i32> 1165 %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 1166 ret <4 x i32> %res 1167} 1168 1169define <8 x i32> @mul_v8i64_zero_upper(<8 x i32> %val1, <8 x i32> %val2) { 1170; SSE2-LABEL: mul_v8i64_zero_upper: 1171; SSE2: # %bb.0: # %entry 1172; SSE2-NEXT: pxor %xmm6, %xmm6 1173; SSE2-NEXT: movdqa %xmm0, %xmm4 1174; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1] 1175; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm6[2],xmm0[3],xmm6[3] 1176; SSE2-NEXT: movdqa %xmm1, %xmm5 1177; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1] 1178; SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm6[2],xmm1[3],xmm6[3] 1179; SSE2-NEXT: movdqa %xmm2, %xmm7 1180; SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1] 1181; SSE2-NEXT: pmuludq %xmm7, %xmm4 1182; SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm6[2],xmm2[3],xmm6[3] 1183; SSE2-NEXT: pmuludq %xmm0, %xmm2 1184; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,3],xmm2[1,3] 1185; SSE2-NEXT: movdqa %xmm3, %xmm0 1186; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1] 1187; SSE2-NEXT: pmuludq %xmm0, %xmm5 1188; SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm6[2],xmm3[3],xmm6[3] 1189; SSE2-NEXT: pmuludq %xmm1, %xmm3 1190; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,3],xmm3[1,3] 1191; SSE2-NEXT: movaps %xmm4, %xmm0 1192; SSE2-NEXT: movaps %xmm5, %xmm1 1193; SSE2-NEXT: retq 1194; 1195; SSE41-LABEL: mul_v8i64_zero_upper: 1196; SSE41: # %bb.0: # %entry 1197; SSE41-NEXT: pxor %xmm6, %xmm6 1198; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero 1199; SSE41-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm6[2],xmm0[3],xmm6[3] 1200; SSE41-NEXT: pmovzxdq {{.*#+}} xmm8 = xmm1[0],zero,xmm1[1],zero 1201; SSE41-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm6[2],xmm1[3],xmm6[3] 1202; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero 1203; SSE41-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm6[2],xmm2[3],xmm6[3] 1204; SSE41-NEXT: pmuludq %xmm0, %xmm2 1205; SSE41-NEXT: pmovzxdq {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero 1206; SSE41-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm6[2],xmm3[3],xmm6[3] 1207; SSE41-NEXT: pmuludq %xmm1, %xmm3 1208; SSE41-NEXT: pmuludq %xmm7, %xmm4 1209; SSE41-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,3],xmm2[1,3] 1210; SSE41-NEXT: pmuludq %xmm8, %xmm5 1211; SSE41-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,3],xmm3[1,3] 1212; SSE41-NEXT: movaps %xmm4, %xmm0 1213; SSE41-NEXT: movaps %xmm5, %xmm1 1214; SSE41-NEXT: retq 1215; 1216; AVX2-LABEL: mul_v8i64_zero_upper: 1217; AVX2: # %bb.0: # %entry 1218; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 1219; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 1220; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 1221; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 1222; AVX2-NEXT: vpmuludq %ymm3, %ymm2, %ymm2 1223; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1 1224; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 1225; AVX2-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 1226; AVX2-NEXT: vshufps {{.*#+}} ymm0 = ymm2[1,3],ymm0[1,3],ymm2[5,7],ymm0[5,7] 1227; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3] 1228; AVX2-NEXT: retq 1229; 1230; AVX512-LABEL: mul_v8i64_zero_upper: 1231; AVX512: # %bb.0: # %entry 1232; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero 1233; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero 1234; AVX512-NEXT: vpmuludq %zmm1, %zmm0, %zmm0 1235; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 1236; AVX512-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7] 1237; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3] 1238; AVX512-NEXT: retq 1239entry: 1240 %val1a = zext <8 x i32> %val1 to <8 x i64> 1241 %val2a = zext <8 x i32> %val2 to <8 x i64> 1242 %res64 = mul <8 x i64> %val1a, %val2a 1243 %rescast = bitcast <8 x i64> %res64 to <16 x i32> 1244 %res = shufflevector <16 x i32> %rescast, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7,i32 9, i32 11, i32 13, i32 15 > 1245 ret <8 x i32> %res 1246} 1247 1248define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) { 1249; SSE2-LABEL: mul_v8i64_sext: 1250; SSE2: # %bb.0: 1251; SSE2-NEXT: movdqa %xmm1, %xmm12 1252; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm0[4],xmm10[5],xmm0[5],xmm10[6],xmm0[6],xmm10[7],xmm0[7] 1253; SSE2-NEXT: psrad $16, %xmm10 1254; SSE2-NEXT: pxor %xmm8, %xmm8 1255; SSE2-NEXT: pxor %xmm13, %xmm13 1256; SSE2-NEXT: pcmpgtd %xmm10, %xmm13 1257; SSE2-NEXT: movdqa %xmm10, %xmm9 1258; SSE2-NEXT: punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm13[2],xmm9[3],xmm13[3] 1259; SSE2-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm13[0],xmm10[1],xmm13[1] 1260; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] 1261; SSE2-NEXT: psrad $16, %xmm0 1262; SSE2-NEXT: pxor %xmm15, %xmm15 1263; SSE2-NEXT: pcmpgtd %xmm0, %xmm15 1264; SSE2-NEXT: movdqa %xmm0, %xmm11 1265; SSE2-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm15[2],xmm11[3],xmm15[3] 1266; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1] 1267; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1] 1268; SSE2-NEXT: pxor %xmm14, %xmm14 1269; SSE2-NEXT: pcmpgtd %xmm3, %xmm14 1270; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm14[0],xmm3[1],xmm14[1] 1271; SSE2-NEXT: pxor %xmm5, %xmm5 1272; SSE2-NEXT: pcmpgtd %xmm2, %xmm5 1273; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1] 1274; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] 1275; SSE2-NEXT: pxor %xmm6, %xmm6 1276; SSE2-NEXT: pcmpgtd %xmm1, %xmm6 1277; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1] 1278; SSE2-NEXT: pxor %xmm7, %xmm7 1279; SSE2-NEXT: pcmpgtd %xmm12, %xmm7 1280; SSE2-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm7[0],xmm12[1],xmm7[1] 1281; SSE2-NEXT: movdqa %xmm15, %xmm4 1282; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1] 1283; SSE2-NEXT: pmuludq %xmm12, %xmm4 1284; SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1] 1285; SSE2-NEXT: pmuludq %xmm0, %xmm7 1286; SSE2-NEXT: paddq %xmm4, %xmm7 1287; SSE2-NEXT: psllq $32, %xmm7 1288; SSE2-NEXT: pmuludq %xmm12, %xmm0 1289; SSE2-NEXT: paddq %xmm7, %xmm0 1290; SSE2-NEXT: punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm8[2],xmm15[3],xmm8[3] 1291; SSE2-NEXT: pmuludq %xmm1, %xmm15 1292; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1] 1293; SSE2-NEXT: pmuludq %xmm11, %xmm6 1294; SSE2-NEXT: paddq %xmm15, %xmm6 1295; SSE2-NEXT: psllq $32, %xmm6 1296; SSE2-NEXT: pmuludq %xmm11, %xmm1 1297; SSE2-NEXT: paddq %xmm6, %xmm1 1298; SSE2-NEXT: movdqa %xmm13, %xmm4 1299; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1] 1300; SSE2-NEXT: pmuludq %xmm2, %xmm4 1301; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1] 1302; SSE2-NEXT: pmuludq %xmm10, %xmm5 1303; SSE2-NEXT: paddq %xmm4, %xmm5 1304; SSE2-NEXT: psllq $32, %xmm5 1305; SSE2-NEXT: pmuludq %xmm10, %xmm2 1306; SSE2-NEXT: paddq %xmm5, %xmm2 1307; SSE2-NEXT: punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm8[2],xmm13[3],xmm8[3] 1308; SSE2-NEXT: pmuludq %xmm3, %xmm13 1309; SSE2-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm8[0],xmm14[1],xmm8[1] 1310; SSE2-NEXT: pmuludq %xmm9, %xmm14 1311; SSE2-NEXT: paddq %xmm13, %xmm14 1312; SSE2-NEXT: psllq $32, %xmm14 1313; SSE2-NEXT: pmuludq %xmm9, %xmm3 1314; SSE2-NEXT: paddq %xmm14, %xmm3 1315; SSE2-NEXT: retq 1316; 1317; SSE41-LABEL: mul_v8i64_sext: 1318; SSE41: # %bb.0: 1319; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3] 1320; SSE41-NEXT: pmovsxwq %xmm3, %xmm4 1321; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1] 1322; SSE41-NEXT: pmovsxwq %xmm3, %xmm5 1323; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3] 1324; SSE41-NEXT: pmovsxwq %xmm3, %xmm6 1325; SSE41-NEXT: pmovsxwq %xmm0, %xmm7 1326; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1] 1327; SSE41-NEXT: pmovsxdq %xmm0, %xmm3 1328; SSE41-NEXT: pmuldq %xmm4, %xmm3 1329; SSE41-NEXT: pmovsxdq %xmm2, %xmm2 1330; SSE41-NEXT: pmuldq %xmm5, %xmm2 1331; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] 1332; SSE41-NEXT: pmovsxdq %xmm0, %xmm4 1333; SSE41-NEXT: pmuldq %xmm6, %xmm4 1334; SSE41-NEXT: pmovsxdq %xmm1, %xmm0 1335; SSE41-NEXT: pmuldq %xmm7, %xmm0 1336; SSE41-NEXT: movdqa %xmm4, %xmm1 1337; SSE41-NEXT: retq 1338; 1339; AVX2-LABEL: mul_v8i64_sext: 1340; AVX2: # %bb.0: 1341; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1] 1342; AVX2-NEXT: vpmovsxwq %xmm2, %ymm2 1343; AVX2-NEXT: vpmovsxwq %xmm0, %ymm0 1344; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3 1345; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero 1346; AVX2-NEXT: vpmuldq %ymm3, %ymm2, %ymm2 1347; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 1348; AVX2-NEXT: vpmuldq %ymm1, %ymm0, %ymm0 1349; AVX2-NEXT: vmovdqa %ymm2, %ymm1 1350; AVX2-NEXT: retq 1351; 1352; AVX512-LABEL: mul_v8i64_sext: 1353; AVX512: # %bb.0: 1354; AVX512-NEXT: vpmovsxwq %xmm0, %zmm0 1355; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero 1356; AVX512-NEXT: vpmuldq %zmm1, %zmm0, %zmm0 1357; AVX512-NEXT: retq 1358 %1 = sext <8 x i16> %val1 to <8 x i64> 1359 %2 = sext <8 x i32> %val2 to <8 x i64> 1360 %3 = mul <8 x i64> %1, %2 1361 ret <8 x i64> %3 1362} 1363 1364define <2 x i64> @pmuldq_square(<2 x i64> %x) { 1365; SSE2-LABEL: pmuldq_square: 1366; SSE2: # %bb.0: 1367; SSE2-NEXT: psllq $32, %xmm0 1368; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3] 1369; SSE2-NEXT: psrad $31, %xmm0 1370; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3] 1371; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] 1372; SSE2-NEXT: psrlq $32, %xmm0 1373; SSE2-NEXT: pmuludq %xmm1, %xmm0 1374; SSE2-NEXT: paddq %xmm0, %xmm0 1375; SSE2-NEXT: psllq $32, %xmm0 1376; SSE2-NEXT: pmuludq %xmm1, %xmm1 1377; SSE2-NEXT: paddq %xmm1, %xmm0 1378; SSE2-NEXT: retq 1379; 1380; SSE41-LABEL: pmuldq_square: 1381; SSE41: # %bb.0: 1382; SSE41-NEXT: pmuldq %xmm0, %xmm0 1383; SSE41-NEXT: retq 1384; 1385; AVX-LABEL: pmuldq_square: 1386; AVX: # %bb.0: 1387; AVX-NEXT: vpmuldq %xmm0, %xmm0, %xmm0 1388; AVX-NEXT: retq 1389 %1 = shl <2 x i64> %x, <i64 32, i64 32> 1390 %2 = ashr exact <2 x i64> %1, <i64 32, i64 32> 1391 %3 = mul nsw <2 x i64> %2, %2 1392 ret <2 x i64> %3 1393} 1394 1395define <2 x i64> @pmuludq_square(<2 x i64> %x) { 1396; SSE-LABEL: pmuludq_square: 1397; SSE: # %bb.0: 1398; SSE-NEXT: pmuludq %xmm0, %xmm0 1399; SSE-NEXT: retq 1400; 1401; AVX-LABEL: pmuludq_square: 1402; AVX: # %bb.0: 1403; AVX-NEXT: vpmuludq %xmm0, %xmm0, %xmm0 1404; AVX-NEXT: retq 1405 %1 = and <2 x i64> %x, <i64 4294967295, i64 4294967295> 1406 %2 = mul nuw <2 x i64> %1, %1 1407 ret <2 x i64> %2 1408} 1409