1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41 4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F 6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW 7 8define <16 x i8> @mul_v16i8c(<16 x i8> %i) nounwind { 9; SSE2-LABEL: mul_v16i8c: 10; SSE2: # %bb.0: # %entry 11; SSE2-NEXT: movdqa %xmm0, %xmm1 12; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] 13; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117] 14; SSE2-NEXT: pmullw %xmm2, %xmm1 15; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255] 16; SSE2-NEXT: pand %xmm3, %xmm1 17; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 18; SSE2-NEXT: pmullw %xmm2, %xmm0 19; SSE2-NEXT: pand %xmm3, %xmm0 20; SSE2-NEXT: packuswb %xmm1, %xmm0 21; SSE2-NEXT: retq 22; 23; SSE41-LABEL: mul_v16i8c: 24; SSE41: # %bb.0: # %entry 25; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 26; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] 27; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117] 28; SSE41-NEXT: pmullw %xmm2, %xmm0 29; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255] 30; SSE41-NEXT: pand %xmm3, %xmm0 31; SSE41-NEXT: pmullw %xmm2, %xmm1 32; SSE41-NEXT: pand %xmm3, %xmm1 33; SSE41-NEXT: packuswb %xmm0, %xmm1 34; SSE41-NEXT: movdqa %xmm1, %xmm0 35; SSE41-NEXT: retq 36; 37; AVX2-LABEL: mul_v16i8c: 38; AVX2: # %bb.0: # %entry 39; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero 40; AVX2-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm0 41; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 42; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 43; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0 44; AVX2-NEXT: vzeroupper 45; AVX2-NEXT: retq 46; 47; AVX512F-LABEL: mul_v16i8c: 48; AVX512F: # %bb.0: # %entry 49; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero 50; AVX512F-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm0 51; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero 52; AVX512F-NEXT: vpmovdb %zmm0, %xmm0 53; AVX512F-NEXT: vzeroupper 54; AVX512F-NEXT: retq 55; 56; AVX512BW-LABEL: mul_v16i8c: 57; AVX512BW: # %bb.0: # %entry 58; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero 59; AVX512BW-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm0 60; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0 61; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 62; AVX512BW-NEXT: vzeroupper 63; AVX512BW-NEXT: retq 64entry: 65 %A = mul <16 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 > 66 ret <16 x i8> %A 67} 68 69define <8 x i16> @mul_v8i16c(<8 x i16> %i) nounwind { 70; SSE-LABEL: mul_v8i16c: 71; SSE: # %bb.0: # %entry 72; SSE-NEXT: pmullw {{.*}}(%rip), %xmm0 73; SSE-NEXT: retq 74; 75; AVX-LABEL: mul_v8i16c: 76; AVX: # %bb.0: # %entry 77; AVX-NEXT: vpmullw {{.*}}(%rip), %xmm0, %xmm0 78; AVX-NEXT: retq 79entry: 80 %A = mul <8 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 > 81 ret <8 x i16> %A 82} 83 84define <4 x i32> @mul_v4i32c(<4 x i32> %i) nounwind { 85; SSE2-LABEL: mul_v4i32c: 86; SSE2: # %bb.0: # %entry 87; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [117,117,117,117] 88; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] 89; SSE2-NEXT: pmuludq %xmm1, %xmm0 90; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] 91; SSE2-NEXT: pmuludq %xmm1, %xmm2 92; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] 93; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] 94; SSE2-NEXT: retq 95; 96; SSE41-LABEL: mul_v4i32c: 97; SSE41: # %bb.0: # %entry 98; SSE41-NEXT: pmulld {{.*}}(%rip), %xmm0 99; SSE41-NEXT: retq 100; 101; AVX-LABEL: mul_v4i32c: 102; AVX: # %bb.0: # %entry 103; AVX-NEXT: vpbroadcastd {{.*#+}} xmm1 = [117,117,117,117] 104; AVX-NEXT: vpmulld %xmm1, %xmm0, %xmm0 105; AVX-NEXT: retq 106entry: 107 %A = mul <4 x i32> %i, < i32 117, i32 117, i32 117, i32 117 > 108 ret <4 x i32> %A 109} 110 111define <2 x i64> @mul_v2i64c(<2 x i64> %i) nounwind { 112; SSE-LABEL: mul_v2i64c: 113; SSE: # %bb.0: # %entry 114; SSE-NEXT: movdqa {{.*#+}} xmm1 = [117,117] 115; SSE-NEXT: movdqa %xmm0, %xmm2 116; SSE-NEXT: pmuludq %xmm1, %xmm2 117; SSE-NEXT: psrlq $32, %xmm0 118; SSE-NEXT: pmuludq %xmm1, %xmm0 119; SSE-NEXT: psllq $32, %xmm0 120; SSE-NEXT: paddq %xmm2, %xmm0 121; SSE-NEXT: retq 122; 123; AVX-LABEL: mul_v2i64c: 124; AVX: # %bb.0: # %entry 125; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [117,117] 126; AVX-NEXT: vpmuludq %xmm1, %xmm0, %xmm2 127; AVX-NEXT: vpsrlq $32, %xmm0, %xmm0 128; AVX-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 129; AVX-NEXT: vpsllq $32, %xmm0, %xmm0 130; AVX-NEXT: vpaddq %xmm0, %xmm2, %xmm0 131; AVX-NEXT: retq 132entry: 133 %A = mul <2 x i64> %i, < i64 117, i64 117 > 134 ret <2 x i64> %A 135} 136 137define <16 x i8> @mul_v16i8(<16 x i8> %i, <16 x i8> %j) nounwind { 138; SSE2-LABEL: mul_v16i8: 139; SSE2: # %bb.0: # %entry 140; SSE2-NEXT: movdqa %xmm1, %xmm2 141; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 142; SSE2-NEXT: movdqa %xmm0, %xmm3 143; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] 144; SSE2-NEXT: pmullw %xmm2, %xmm3 145; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255] 146; SSE2-NEXT: pand %xmm2, %xmm3 147; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 148; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 149; SSE2-NEXT: pmullw %xmm1, %xmm0 150; SSE2-NEXT: pand %xmm2, %xmm0 151; SSE2-NEXT: packuswb %xmm3, %xmm0 152; SSE2-NEXT: retq 153; 154; SSE41-LABEL: mul_v16i8: 155; SSE41: # %bb.0: # %entry 156; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 157; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] 158; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 159; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] 160; SSE41-NEXT: pmullw %xmm1, %xmm0 161; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] 162; SSE41-NEXT: pand %xmm1, %xmm0 163; SSE41-NEXT: pmullw %xmm3, %xmm2 164; SSE41-NEXT: pand %xmm1, %xmm2 165; SSE41-NEXT: packuswb %xmm0, %xmm2 166; SSE41-NEXT: movdqa %xmm2, %xmm0 167; SSE41-NEXT: retq 168; 169; AVX2-LABEL: mul_v16i8: 170; AVX2: # %bb.0: # %entry 171; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero 172; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero 173; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0 174; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 175; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 176; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0 177; AVX2-NEXT: vzeroupper 178; AVX2-NEXT: retq 179; 180; AVX512F-LABEL: mul_v16i8: 181; AVX512F: # %bb.0: # %entry 182; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero 183; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero 184; AVX512F-NEXT: vpmullw %ymm1, %ymm0, %ymm0 185; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero 186; AVX512F-NEXT: vpmovdb %zmm0, %xmm0 187; AVX512F-NEXT: vzeroupper 188; AVX512F-NEXT: retq 189; 190; AVX512BW-LABEL: mul_v16i8: 191; AVX512BW: # %bb.0: # %entry 192; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero 193; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero 194; AVX512BW-NEXT: vpmullw %ymm1, %ymm0, %ymm0 195; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0 196; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 197; AVX512BW-NEXT: vzeroupper 198; AVX512BW-NEXT: retq 199entry: 200 %A = mul <16 x i8> %i, %j 201 ret <16 x i8> %A 202} 203 204define <8 x i16> @mul_v8i16(<8 x i16> %i, <8 x i16> %j) nounwind { 205; SSE-LABEL: mul_v8i16: 206; SSE: # %bb.0: # %entry 207; SSE-NEXT: pmullw %xmm1, %xmm0 208; SSE-NEXT: retq 209; 210; AVX-LABEL: mul_v8i16: 211; AVX: # %bb.0: # %entry 212; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0 213; AVX-NEXT: retq 214entry: 215 %A = mul <8 x i16> %i, %j 216 ret <8 x i16> %A 217} 218 219define <4 x i32> @mul_v4i32(<4 x i32> %i, <4 x i32> %j) nounwind { 220; SSE2-LABEL: mul_v4i32: 221; SSE2: # %bb.0: # %entry 222; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] 223; SSE2-NEXT: pmuludq %xmm1, %xmm0 224; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] 225; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] 226; SSE2-NEXT: pmuludq %xmm2, %xmm1 227; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] 228; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] 229; SSE2-NEXT: retq 230; 231; SSE41-LABEL: mul_v4i32: 232; SSE41: # %bb.0: # %entry 233; SSE41-NEXT: pmulld %xmm1, %xmm0 234; SSE41-NEXT: retq 235; 236; AVX-LABEL: mul_v4i32: 237; AVX: # %bb.0: # %entry 238; AVX-NEXT: vpmulld %xmm1, %xmm0, %xmm0 239; AVX-NEXT: retq 240entry: 241 %A = mul <4 x i32> %i, %j 242 ret <4 x i32> %A 243} 244 245define <2 x i64> @mul_v2i64(<2 x i64> %i, <2 x i64> %j) nounwind { 246; SSE-LABEL: mul_v2i64: 247; SSE: # %bb.0: # %entry 248; SSE-NEXT: movdqa %xmm0, %xmm2 249; SSE-NEXT: psrlq $32, %xmm2 250; SSE-NEXT: pmuludq %xmm1, %xmm2 251; SSE-NEXT: movdqa %xmm1, %xmm3 252; SSE-NEXT: psrlq $32, %xmm3 253; SSE-NEXT: pmuludq %xmm0, %xmm3 254; SSE-NEXT: paddq %xmm2, %xmm3 255; SSE-NEXT: psllq $32, %xmm3 256; SSE-NEXT: pmuludq %xmm1, %xmm0 257; SSE-NEXT: paddq %xmm3, %xmm0 258; SSE-NEXT: retq 259; 260; AVX-LABEL: mul_v2i64: 261; AVX: # %bb.0: # %entry 262; AVX-NEXT: vpsrlq $32, %xmm0, %xmm2 263; AVX-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 264; AVX-NEXT: vpsrlq $32, %xmm1, %xmm3 265; AVX-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 266; AVX-NEXT: vpaddq %xmm2, %xmm3, %xmm2 267; AVX-NEXT: vpsllq $32, %xmm2, %xmm2 268; AVX-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 269; AVX-NEXT: vpaddq %xmm2, %xmm0, %xmm0 270; AVX-NEXT: retq 271entry: 272 %A = mul <2 x i64> %i, %j 273 ret <2 x i64> %A 274} 275 276declare void @foo() 277 278define <4 x i32> @mul_v4i32spill(<4 x i32> %i, <4 x i32> %j) nounwind { 279; SSE2-LABEL: mul_v4i32spill: 280; SSE2: # %bb.0: # %entry 281; SSE2-NEXT: subq $40, %rsp 282; SSE2-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 283; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill 284; SSE2-NEXT: callq foo 285; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload 286; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] 287; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload 288; SSE2-NEXT: pmuludq %xmm2, %xmm0 289; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] 290; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] 291; SSE2-NEXT: pmuludq %xmm1, %xmm2 292; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] 293; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] 294; SSE2-NEXT: addq $40, %rsp 295; SSE2-NEXT: retq 296; 297; SSE41-LABEL: mul_v4i32spill: 298; SSE41: # %bb.0: # %entry 299; SSE41-NEXT: subq $40, %rsp 300; SSE41-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 301; SSE41-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill 302; SSE41-NEXT: callq foo 303; SSE41-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload 304; SSE41-NEXT: pmulld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload 305; SSE41-NEXT: addq $40, %rsp 306; SSE41-NEXT: retq 307; 308; AVX-LABEL: mul_v4i32spill: 309; AVX: # %bb.0: # %entry 310; AVX-NEXT: subq $40, %rsp 311; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 312; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill 313; AVX-NEXT: callq foo 314; AVX-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload 315; AVX-NEXT: vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload 316; AVX-NEXT: addq $40, %rsp 317; AVX-NEXT: retq 318entry: 319 ; Use a call to force spills. 320 call void @foo() 321 %A = mul <4 x i32> %i, %j 322 ret <4 x i32> %A 323} 324 325define <2 x i64> @mul_v2i64spill(<2 x i64> %i, <2 x i64> %j) nounwind { 326; SSE-LABEL: mul_v2i64spill: 327; SSE: # %bb.0: # %entry 328; SSE-NEXT: subq $40, %rsp 329; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 330; SSE-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill 331; SSE-NEXT: callq foo 332; SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload 333; SSE-NEXT: movdqa %xmm0, %xmm2 334; SSE-NEXT: psrlq $32, %xmm2 335; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload 336; SSE-NEXT: pmuludq %xmm3, %xmm2 337; SSE-NEXT: movdqa %xmm3, %xmm1 338; SSE-NEXT: psrlq $32, %xmm1 339; SSE-NEXT: pmuludq %xmm0, %xmm1 340; SSE-NEXT: paddq %xmm2, %xmm1 341; SSE-NEXT: psllq $32, %xmm1 342; SSE-NEXT: pmuludq %xmm3, %xmm0 343; SSE-NEXT: paddq %xmm1, %xmm0 344; SSE-NEXT: addq $40, %rsp 345; SSE-NEXT: retq 346; 347; AVX-LABEL: mul_v2i64spill: 348; AVX: # %bb.0: # %entry 349; AVX-NEXT: subq $40, %rsp 350; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 351; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill 352; AVX-NEXT: callq foo 353; AVX-NEXT: vmovdqa (%rsp), %xmm3 # 16-byte Reload 354; AVX-NEXT: vpsrlq $32, %xmm3, %xmm0 355; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload 356; AVX-NEXT: vpmuludq %xmm2, %xmm0, %xmm0 357; AVX-NEXT: vpsrlq $32, %xmm2, %xmm1 358; AVX-NEXT: vpmuludq %xmm1, %xmm3, %xmm1 359; AVX-NEXT: vpaddq %xmm0, %xmm1, %xmm0 360; AVX-NEXT: vpsllq $32, %xmm0, %xmm0 361; AVX-NEXT: vpmuludq %xmm2, %xmm3, %xmm1 362; AVX-NEXT: vpaddq %xmm0, %xmm1, %xmm0 363; AVX-NEXT: addq $40, %rsp 364; AVX-NEXT: retq 365entry: 366 ; Use a call to force spills. 367 call void @foo() 368 %A = mul <2 x i64> %i, %j 369 ret <2 x i64> %A 370} 371 372define <32 x i8> @mul_v32i8c(<32 x i8> %i) nounwind { 373; SSE2-LABEL: mul_v32i8c: 374; SSE2: # %bb.0: # %entry 375; SSE2-NEXT: movdqa %xmm0, %xmm2 376; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 377; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [117,117,117,117,117,117,117,117] 378; SSE2-NEXT: pmullw %xmm3, %xmm2 379; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255] 380; SSE2-NEXT: pand %xmm4, %xmm2 381; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 382; SSE2-NEXT: pmullw %xmm3, %xmm0 383; SSE2-NEXT: pand %xmm4, %xmm0 384; SSE2-NEXT: packuswb %xmm2, %xmm0 385; SSE2-NEXT: movdqa %xmm1, %xmm2 386; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 387; SSE2-NEXT: pmullw %xmm3, %xmm2 388; SSE2-NEXT: pand %xmm4, %xmm2 389; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 390; SSE2-NEXT: pmullw %xmm3, %xmm1 391; SSE2-NEXT: pand %xmm4, %xmm1 392; SSE2-NEXT: packuswb %xmm2, %xmm1 393; SSE2-NEXT: retq 394; 395; SSE41-LABEL: mul_v32i8c: 396; SSE41: # %bb.0: # %entry 397; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 398; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] 399; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [117,117,117,117,117,117,117,117] 400; SSE41-NEXT: pmullw %xmm4, %xmm0 401; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255] 402; SSE41-NEXT: pand %xmm5, %xmm0 403; SSE41-NEXT: pmullw %xmm4, %xmm2 404; SSE41-NEXT: pand %xmm5, %xmm2 405; SSE41-NEXT: packuswb %xmm0, %xmm2 406; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 407; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] 408; SSE41-NEXT: pmullw %xmm4, %xmm1 409; SSE41-NEXT: pand %xmm5, %xmm1 410; SSE41-NEXT: pmullw %xmm4, %xmm3 411; SSE41-NEXT: pand %xmm5, %xmm3 412; SSE41-NEXT: packuswb %xmm1, %xmm3 413; SSE41-NEXT: movdqa %xmm2, %xmm0 414; SSE41-NEXT: movdqa %xmm3, %xmm1 415; SSE41-NEXT: retq 416; 417; AVX2-LABEL: mul_v32i8c: 418; AVX2: # %bb.0: # %entry 419; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 420; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117] 421; AVX2-NEXT: vpmullw %ymm2, %ymm1, %ymm1 422; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 423; AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1 424; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 425; AVX2-NEXT: vpmullw %ymm2, %ymm0, %ymm0 426; AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0 427; AVX2-NEXT: vpackuswb %ymm1, %ymm0, %ymm0 428; AVX2-NEXT: retq 429; 430; AVX512F-LABEL: mul_v32i8c: 431; AVX512F: # %bb.0: # %entry 432; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 433; AVX512F-NEXT: vmovdqa {{.*#+}} ymm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117] 434; AVX512F-NEXT: vpmullw %ymm2, %ymm1, %ymm1 435; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 436; AVX512F-NEXT: vpand %ymm3, %ymm1, %ymm1 437; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 438; AVX512F-NEXT: vpmullw %ymm2, %ymm0, %ymm0 439; AVX512F-NEXT: vpand %ymm3, %ymm0, %ymm0 440; AVX512F-NEXT: vpackuswb %ymm1, %ymm0, %ymm0 441; AVX512F-NEXT: retq 442; 443; AVX512BW-LABEL: mul_v32i8c: 444; AVX512BW: # %bb.0: # %entry 445; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero 446; AVX512BW-NEXT: vpmullw {{.*}}(%rip), %zmm0, %zmm0 447; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0 448; AVX512BW-NEXT: retq 449entry: 450 %A = mul <32 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 > 451 ret <32 x i8> %A 452} 453 454define <16 x i16> @mul_v16i16c(<16 x i16> %i) nounwind { 455; SSE-LABEL: mul_v16i16c: 456; SSE: # %bb.0: # %entry 457; SSE-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117] 458; SSE-NEXT: pmullw %xmm2, %xmm0 459; SSE-NEXT: pmullw %xmm2, %xmm1 460; SSE-NEXT: retq 461; 462; AVX-LABEL: mul_v16i16c: 463; AVX: # %bb.0: # %entry 464; AVX-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm0 465; AVX-NEXT: retq 466entry: 467 %A = mul <16 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 > 468 ret <16 x i16> %A 469} 470 471define <8 x i32> @mul_v8i32c(<8 x i32> %i) nounwind { 472; SSE2-LABEL: mul_v8i32c: 473; SSE2: # %bb.0: # %entry 474; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117] 475; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] 476; SSE2-NEXT: pmuludq %xmm2, %xmm0 477; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] 478; SSE2-NEXT: pmuludq %xmm2, %xmm3 479; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] 480; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] 481; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3] 482; SSE2-NEXT: pmuludq %xmm2, %xmm1 483; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] 484; SSE2-NEXT: pmuludq %xmm2, %xmm3 485; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3] 486; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] 487; SSE2-NEXT: retq 488; 489; SSE41-LABEL: mul_v8i32c: 490; SSE41: # %bb.0: # %entry 491; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117] 492; SSE41-NEXT: pmulld %xmm2, %xmm0 493; SSE41-NEXT: pmulld %xmm2, %xmm1 494; SSE41-NEXT: retq 495; 496; AVX-LABEL: mul_v8i32c: 497; AVX: # %bb.0: # %entry 498; AVX-NEXT: vpbroadcastd {{.*#+}} ymm1 = [117,117,117,117,117,117,117,117] 499; AVX-NEXT: vpmulld %ymm1, %ymm0, %ymm0 500; AVX-NEXT: retq 501entry: 502 %A = mul <8 x i32> %i, < i32 117, i32 117, i32 117, i32 117, i32 117, i32 117, i32 117, i32 117 > 503 ret <8 x i32> %A 504} 505 506define <4 x i64> @mul_v4i64c(<4 x i64> %i) nounwind { 507; SSE-LABEL: mul_v4i64c: 508; SSE: # %bb.0: # %entry 509; SSE-NEXT: movdqa {{.*#+}} xmm2 = [117,117] 510; SSE-NEXT: movdqa %xmm0, %xmm3 511; SSE-NEXT: pmuludq %xmm2, %xmm3 512; SSE-NEXT: psrlq $32, %xmm0 513; SSE-NEXT: pmuludq %xmm2, %xmm0 514; SSE-NEXT: psllq $32, %xmm0 515; SSE-NEXT: paddq %xmm3, %xmm0 516; SSE-NEXT: movdqa %xmm1, %xmm3 517; SSE-NEXT: pmuludq %xmm2, %xmm3 518; SSE-NEXT: psrlq $32, %xmm1 519; SSE-NEXT: pmuludq %xmm2, %xmm1 520; SSE-NEXT: psllq $32, %xmm1 521; SSE-NEXT: paddq %xmm3, %xmm1 522; SSE-NEXT: retq 523; 524; AVX-LABEL: mul_v4i64c: 525; AVX: # %bb.0: # %entry 526; AVX-NEXT: vpbroadcastq {{.*#+}} ymm1 = [117,117,117,117] 527; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm2 528; AVX-NEXT: vpsrlq $32, %ymm0, %ymm0 529; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 530; AVX-NEXT: vpsllq $32, %ymm0, %ymm0 531; AVX-NEXT: vpaddq %ymm0, %ymm2, %ymm0 532; AVX-NEXT: retq 533entry: 534 %A = mul <4 x i64> %i, < i64 117, i64 117, i64 117, i64 117 > 535 ret <4 x i64> %A 536} 537 538define <32 x i8> @mul_v32i8(<32 x i8> %i, <32 x i8> %j) nounwind { 539; SSE2-LABEL: mul_v32i8: 540; SSE2: # %bb.0: # %entry 541; SSE2-NEXT: movdqa %xmm2, %xmm4 542; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] 543; SSE2-NEXT: movdqa %xmm0, %xmm5 544; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15] 545; SSE2-NEXT: pmullw %xmm4, %xmm5 546; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255] 547; SSE2-NEXT: pand %xmm4, %xmm5 548; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] 549; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 550; SSE2-NEXT: pmullw %xmm2, %xmm0 551; SSE2-NEXT: pand %xmm4, %xmm0 552; SSE2-NEXT: packuswb %xmm5, %xmm0 553; SSE2-NEXT: movdqa %xmm3, %xmm2 554; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 555; SSE2-NEXT: movdqa %xmm1, %xmm5 556; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15] 557; SSE2-NEXT: pmullw %xmm2, %xmm5 558; SSE2-NEXT: pand %xmm4, %xmm5 559; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 560; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 561; SSE2-NEXT: pmullw %xmm3, %xmm1 562; SSE2-NEXT: pand %xmm4, %xmm1 563; SSE2-NEXT: packuswb %xmm5, %xmm1 564; SSE2-NEXT: retq 565; 566; SSE41-LABEL: mul_v32i8: 567; SSE41: # %bb.0: # %entry 568; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero 569; SSE41-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 570; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 571; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] 572; SSE41-NEXT: pmullw %xmm2, %xmm0 573; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255] 574; SSE41-NEXT: pand %xmm6, %xmm0 575; SSE41-NEXT: pmullw %xmm5, %xmm4 576; SSE41-NEXT: pand %xmm6, %xmm4 577; SSE41-NEXT: packuswb %xmm0, %xmm4 578; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero 579; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] 580; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 581; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] 582; SSE41-NEXT: pmullw %xmm3, %xmm1 583; SSE41-NEXT: pand %xmm6, %xmm1 584; SSE41-NEXT: pmullw %xmm0, %xmm2 585; SSE41-NEXT: pand %xmm6, %xmm2 586; SSE41-NEXT: packuswb %xmm1, %xmm2 587; SSE41-NEXT: movdqa %xmm4, %xmm0 588; SSE41-NEXT: movdqa %xmm2, %xmm1 589; SSE41-NEXT: retq 590; 591; AVX2-LABEL: mul_v32i8: 592; AVX2: # %bb.0: # %entry 593; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31] 594; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 595; AVX2-NEXT: vpmullw %ymm2, %ymm3, %ymm2 596; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 597; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2 598; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23] 599; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 600; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0 601; AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0 602; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 603; AVX2-NEXT: retq 604; 605; AVX512F-LABEL: mul_v32i8: 606; AVX512F: # %bb.0: # %entry 607; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31] 608; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 609; AVX512F-NEXT: vpmullw %ymm2, %ymm3, %ymm2 610; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 611; AVX512F-NEXT: vpand %ymm3, %ymm2, %ymm2 612; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23] 613; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 614; AVX512F-NEXT: vpmullw %ymm1, %ymm0, %ymm0 615; AVX512F-NEXT: vpand %ymm3, %ymm0, %ymm0 616; AVX512F-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 617; AVX512F-NEXT: retq 618; 619; AVX512BW-LABEL: mul_v32i8: 620; AVX512BW: # %bb.0: # %entry 621; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero 622; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero 623; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0 624; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0 625; AVX512BW-NEXT: retq 626entry: 627 %A = mul <32 x i8> %i, %j 628 ret <32 x i8> %A 629} 630 631define <16 x i16> @mul_v16i16(<16 x i16> %i, <16 x i16> %j) nounwind { 632; SSE-LABEL: mul_v16i16: 633; SSE: # %bb.0: # %entry 634; SSE-NEXT: pmullw %xmm2, %xmm0 635; SSE-NEXT: pmullw %xmm3, %xmm1 636; SSE-NEXT: retq 637; 638; AVX-LABEL: mul_v16i16: 639; AVX: # %bb.0: # %entry 640; AVX-NEXT: vpmullw %ymm1, %ymm0, %ymm0 641; AVX-NEXT: retq 642entry: 643 %A = mul <16 x i16> %i, %j 644 ret <16 x i16> %A 645} 646 647define <8 x i32> @mul_v8i32(<8 x i32> %i, <8 x i32> %j) nounwind { 648; SSE2-LABEL: mul_v8i32: 649; SSE2: # %bb.0: # %entry 650; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] 651; SSE2-NEXT: pmuludq %xmm2, %xmm0 652; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] 653; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] 654; SSE2-NEXT: pmuludq %xmm4, %xmm2 655; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] 656; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] 657; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3] 658; SSE2-NEXT: pmuludq %xmm3, %xmm1 659; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] 660; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3] 661; SSE2-NEXT: pmuludq %xmm2, %xmm3 662; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3] 663; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] 664; SSE2-NEXT: retq 665; 666; SSE41-LABEL: mul_v8i32: 667; SSE41: # %bb.0: # %entry 668; SSE41-NEXT: pmulld %xmm2, %xmm0 669; SSE41-NEXT: pmulld %xmm3, %xmm1 670; SSE41-NEXT: retq 671; 672; AVX-LABEL: mul_v8i32: 673; AVX: # %bb.0: # %entry 674; AVX-NEXT: vpmulld %ymm1, %ymm0, %ymm0 675; AVX-NEXT: retq 676entry: 677 %A = mul <8 x i32> %i, %j 678 ret <8 x i32> %A 679} 680 681define <4 x i64> @mul_v4i64(<4 x i64> %i, <4 x i64> %j) nounwind { 682; SSE-LABEL: mul_v4i64: 683; SSE: # %bb.0: # %entry 684; SSE-NEXT: movdqa %xmm0, %xmm4 685; SSE-NEXT: psrlq $32, %xmm4 686; SSE-NEXT: pmuludq %xmm2, %xmm4 687; SSE-NEXT: movdqa %xmm2, %xmm5 688; SSE-NEXT: psrlq $32, %xmm5 689; SSE-NEXT: pmuludq %xmm0, %xmm5 690; SSE-NEXT: paddq %xmm4, %xmm5 691; SSE-NEXT: psllq $32, %xmm5 692; SSE-NEXT: pmuludq %xmm2, %xmm0 693; SSE-NEXT: paddq %xmm5, %xmm0 694; SSE-NEXT: movdqa %xmm1, %xmm2 695; SSE-NEXT: psrlq $32, %xmm2 696; SSE-NEXT: pmuludq %xmm3, %xmm2 697; SSE-NEXT: movdqa %xmm3, %xmm4 698; SSE-NEXT: psrlq $32, %xmm4 699; SSE-NEXT: pmuludq %xmm1, %xmm4 700; SSE-NEXT: paddq %xmm2, %xmm4 701; SSE-NEXT: psllq $32, %xmm4 702; SSE-NEXT: pmuludq %xmm3, %xmm1 703; SSE-NEXT: paddq %xmm4, %xmm1 704; SSE-NEXT: retq 705; 706; AVX-LABEL: mul_v4i64: 707; AVX: # %bb.0: # %entry 708; AVX-NEXT: vpsrlq $32, %ymm0, %ymm2 709; AVX-NEXT: vpmuludq %ymm1, %ymm2, %ymm2 710; AVX-NEXT: vpsrlq $32, %ymm1, %ymm3 711; AVX-NEXT: vpmuludq %ymm3, %ymm0, %ymm3 712; AVX-NEXT: vpaddq %ymm2, %ymm3, %ymm2 713; AVX-NEXT: vpsllq $32, %ymm2, %ymm2 714; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 715; AVX-NEXT: vpaddq %ymm2, %ymm0, %ymm0 716; AVX-NEXT: retq 717entry: 718 %A = mul <4 x i64> %i, %j 719 ret <4 x i64> %A 720} 721 722define <64 x i8> @mul_v64i8c(<64 x i8> %i) nounwind { 723; SSE2-LABEL: mul_v64i8c: 724; SSE2: # %bb.0: # %entry 725; SSE2-NEXT: movdqa %xmm0, %xmm6 726; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15] 727; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [117,117,117,117,117,117,117,117] 728; SSE2-NEXT: pmullw %xmm4, %xmm6 729; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255] 730; SSE2-NEXT: pand %xmm5, %xmm6 731; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 732; SSE2-NEXT: pmullw %xmm4, %xmm0 733; SSE2-NEXT: pand %xmm5, %xmm0 734; SSE2-NEXT: packuswb %xmm6, %xmm0 735; SSE2-NEXT: movdqa %xmm1, %xmm6 736; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15] 737; SSE2-NEXT: pmullw %xmm4, %xmm6 738; SSE2-NEXT: pand %xmm5, %xmm6 739; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 740; SSE2-NEXT: pmullw %xmm4, %xmm1 741; SSE2-NEXT: pand %xmm5, %xmm1 742; SSE2-NEXT: packuswb %xmm6, %xmm1 743; SSE2-NEXT: movdqa %xmm2, %xmm6 744; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15] 745; SSE2-NEXT: pmullw %xmm4, %xmm6 746; SSE2-NEXT: pand %xmm5, %xmm6 747; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] 748; SSE2-NEXT: pmullw %xmm4, %xmm2 749; SSE2-NEXT: pand %xmm5, %xmm2 750; SSE2-NEXT: packuswb %xmm6, %xmm2 751; SSE2-NEXT: movdqa %xmm3, %xmm6 752; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15] 753; SSE2-NEXT: pmullw %xmm4, %xmm6 754; SSE2-NEXT: pand %xmm5, %xmm6 755; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 756; SSE2-NEXT: pmullw %xmm4, %xmm3 757; SSE2-NEXT: pand %xmm5, %xmm3 758; SSE2-NEXT: packuswb %xmm6, %xmm3 759; SSE2-NEXT: retq 760; 761; SSE41-LABEL: mul_v64i8c: 762; SSE41: # %bb.0: # %entry 763; SSE41-NEXT: movdqa %xmm1, %xmm4 764; SSE41-NEXT: movdqa %xmm0, %xmm1 765; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 766; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] 767; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [117,117,117,117,117,117,117,117] 768; SSE41-NEXT: pmullw %xmm6, %xmm1 769; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255] 770; SSE41-NEXT: pand %xmm7, %xmm1 771; SSE41-NEXT: pmullw %xmm6, %xmm0 772; SSE41-NEXT: pand %xmm7, %xmm0 773; SSE41-NEXT: packuswb %xmm1, %xmm0 774; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero 775; SSE41-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] 776; SSE41-NEXT: pmullw %xmm6, %xmm4 777; SSE41-NEXT: pand %xmm7, %xmm4 778; SSE41-NEXT: pmullw %xmm6, %xmm1 779; SSE41-NEXT: pand %xmm7, %xmm1 780; SSE41-NEXT: packuswb %xmm4, %xmm1 781; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero 782; SSE41-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 783; SSE41-NEXT: pmullw %xmm6, %xmm2 784; SSE41-NEXT: pand %xmm7, %xmm2 785; SSE41-NEXT: pmullw %xmm6, %xmm4 786; SSE41-NEXT: pand %xmm7, %xmm4 787; SSE41-NEXT: packuswb %xmm2, %xmm4 788; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero 789; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] 790; SSE41-NEXT: pmullw %xmm6, %xmm3 791; SSE41-NEXT: pand %xmm7, %xmm3 792; SSE41-NEXT: pmullw %xmm6, %xmm5 793; SSE41-NEXT: pand %xmm7, %xmm5 794; SSE41-NEXT: packuswb %xmm3, %xmm5 795; SSE41-NEXT: movdqa %xmm4, %xmm2 796; SSE41-NEXT: movdqa %xmm5, %xmm3 797; SSE41-NEXT: retq 798; 799; AVX2-LABEL: mul_v64i8c: 800; AVX2: # %bb.0: # %entry 801; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 802; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117] 803; AVX2-NEXT: vpmullw %ymm3, %ymm2, %ymm2 804; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 805; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2 806; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 807; AVX2-NEXT: vpmullw %ymm3, %ymm0, %ymm0 808; AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0 809; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 810; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31] 811; AVX2-NEXT: vpmullw %ymm3, %ymm2, %ymm2 812; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2 813; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23] 814; AVX2-NEXT: vpmullw %ymm3, %ymm1, %ymm1 815; AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1 816; AVX2-NEXT: vpackuswb %ymm2, %ymm1, %ymm1 817; AVX2-NEXT: retq 818; 819; AVX512F-LABEL: mul_v64i8c: 820; AVX512F: # %bb.0: # %entry 821; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 822; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117] 823; AVX512F-NEXT: vpmullw %ymm3, %ymm2, %ymm2 824; AVX512F-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 825; AVX512F-NEXT: vpand %ymm4, %ymm2, %ymm2 826; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 827; AVX512F-NEXT: vpmullw %ymm3, %ymm0, %ymm0 828; AVX512F-NEXT: vpand %ymm4, %ymm0, %ymm0 829; AVX512F-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 830; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31] 831; AVX512F-NEXT: vpmullw %ymm3, %ymm2, %ymm2 832; AVX512F-NEXT: vpand %ymm4, %ymm2, %ymm2 833; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23] 834; AVX512F-NEXT: vpmullw %ymm3, %ymm1, %ymm1 835; AVX512F-NEXT: vpand %ymm4, %ymm1, %ymm1 836; AVX512F-NEXT: vpackuswb %ymm2, %ymm1, %ymm1 837; AVX512F-NEXT: retq 838; 839; AVX512BW-LABEL: mul_v64i8c: 840; AVX512BW: # %bb.0: # %entry 841; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] 842; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117] 843; AVX512BW-NEXT: vpmullw %zmm2, %zmm1, %zmm1 844; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 845; AVX512BW-NEXT: vpandq %zmm3, %zmm1, %zmm1 846; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55] 847; AVX512BW-NEXT: vpmullw %zmm2, %zmm0, %zmm0 848; AVX512BW-NEXT: vpandq %zmm3, %zmm0, %zmm0 849; AVX512BW-NEXT: vpackuswb %zmm1, %zmm0, %zmm0 850; AVX512BW-NEXT: retq 851entry: 852 %A = mul <64 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 > 853 ret <64 x i8> %A 854} 855 856define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind { 857; SSE2-LABEL: mul_v64i8: 858; SSE2: # %bb.0: # %entry 859; SSE2-NEXT: movdqa %xmm4, %xmm8 860; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15] 861; SSE2-NEXT: movdqa %xmm0, %xmm9 862; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm0[8],xmm9[9],xmm0[9],xmm9[10],xmm0[10],xmm9[11],xmm0[11],xmm9[12],xmm0[12],xmm9[13],xmm0[13],xmm9[14],xmm0[14],xmm9[15],xmm0[15] 863; SSE2-NEXT: pmullw %xmm8, %xmm9 864; SSE2-NEXT: movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255] 865; SSE2-NEXT: pand %xmm8, %xmm9 866; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 867; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 868; SSE2-NEXT: pmullw %xmm4, %xmm0 869; SSE2-NEXT: pand %xmm8, %xmm0 870; SSE2-NEXT: packuswb %xmm9, %xmm0 871; SSE2-NEXT: movdqa %xmm5, %xmm9 872; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm0[8],xmm9[9],xmm0[9],xmm9[10],xmm0[10],xmm9[11],xmm0[11],xmm9[12],xmm0[12],xmm9[13],xmm0[13],xmm9[14],xmm0[14],xmm9[15],xmm0[15] 873; SSE2-NEXT: movdqa %xmm1, %xmm4 874; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] 875; SSE2-NEXT: pmullw %xmm9, %xmm4 876; SSE2-NEXT: pand %xmm8, %xmm4 877; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7] 878; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 879; SSE2-NEXT: pmullw %xmm5, %xmm1 880; SSE2-NEXT: pand %xmm8, %xmm1 881; SSE2-NEXT: packuswb %xmm4, %xmm1 882; SSE2-NEXT: movdqa %xmm6, %xmm4 883; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] 884; SSE2-NEXT: movdqa %xmm2, %xmm5 885; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15] 886; SSE2-NEXT: pmullw %xmm4, %xmm5 887; SSE2-NEXT: pand %xmm8, %xmm5 888; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7] 889; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] 890; SSE2-NEXT: pmullw %xmm6, %xmm2 891; SSE2-NEXT: pand %xmm8, %xmm2 892; SSE2-NEXT: packuswb %xmm5, %xmm2 893; SSE2-NEXT: movdqa %xmm7, %xmm4 894; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] 895; SSE2-NEXT: movdqa %xmm3, %xmm5 896; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15] 897; SSE2-NEXT: pmullw %xmm4, %xmm5 898; SSE2-NEXT: pand %xmm8, %xmm5 899; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7] 900; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 901; SSE2-NEXT: pmullw %xmm7, %xmm3 902; SSE2-NEXT: pand %xmm8, %xmm3 903; SSE2-NEXT: packuswb %xmm5, %xmm3 904; SSE2-NEXT: retq 905; 906; SSE41-LABEL: mul_v64i8: 907; SSE41: # %bb.0: # %entry 908; SSE41-NEXT: movdqa %xmm1, %xmm8 909; SSE41-NEXT: movdqa %xmm0, %xmm1 910; SSE41-NEXT: pmovzxbw {{.*#+}} xmm10 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero 911; SSE41-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] 912; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 913; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] 914; SSE41-NEXT: pmullw %xmm4, %xmm1 915; SSE41-NEXT: movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255] 916; SSE41-NEXT: pand %xmm9, %xmm1 917; SSE41-NEXT: pmullw %xmm10, %xmm0 918; SSE41-NEXT: pand %xmm9, %xmm0 919; SSE41-NEXT: packuswb %xmm1, %xmm0 920; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero 921; SSE41-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15] 922; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero,xmm8[4],zero,xmm8[5],zero,xmm8[6],zero,xmm8[7],zero 923; SSE41-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15] 924; SSE41-NEXT: pmullw %xmm5, %xmm8 925; SSE41-NEXT: pand %xmm9, %xmm8 926; SSE41-NEXT: pmullw %xmm4, %xmm1 927; SSE41-NEXT: pand %xmm9, %xmm1 928; SSE41-NEXT: packuswb %xmm8, %xmm1 929; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero 930; SSE41-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15] 931; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero 932; SSE41-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] 933; SSE41-NEXT: pmullw %xmm6, %xmm2 934; SSE41-NEXT: pand %xmm9, %xmm2 935; SSE41-NEXT: pmullw %xmm5, %xmm4 936; SSE41-NEXT: pand %xmm9, %xmm4 937; SSE41-NEXT: packuswb %xmm2, %xmm4 938; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero 939; SSE41-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15] 940; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero 941; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] 942; SSE41-NEXT: pmullw %xmm7, %xmm3 943; SSE41-NEXT: pand %xmm9, %xmm3 944; SSE41-NEXT: pmullw %xmm2, %xmm5 945; SSE41-NEXT: pand %xmm9, %xmm5 946; SSE41-NEXT: packuswb %xmm3, %xmm5 947; SSE41-NEXT: movdqa %xmm4, %xmm2 948; SSE41-NEXT: movdqa %xmm5, %xmm3 949; SSE41-NEXT: retq 950; 951; AVX2-LABEL: mul_v64i8: 952; AVX2: # %bb.0: # %entry 953; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31] 954; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 955; AVX2-NEXT: vpmullw %ymm4, %ymm5, %ymm4 956; AVX2-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 957; AVX2-NEXT: vpand %ymm5, %ymm4, %ymm4 958; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23] 959; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 960; AVX2-NEXT: vpmullw %ymm2, %ymm0, %ymm0 961; AVX2-NEXT: vpand %ymm5, %ymm0, %ymm0 962; AVX2-NEXT: vpackuswb %ymm4, %ymm0, %ymm0 963; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31] 964; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31] 965; AVX2-NEXT: vpmullw %ymm2, %ymm4, %ymm2 966; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2 967; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23] 968; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23] 969; AVX2-NEXT: vpmullw %ymm3, %ymm1, %ymm1 970; AVX2-NEXT: vpand %ymm5, %ymm1, %ymm1 971; AVX2-NEXT: vpackuswb %ymm2, %ymm1, %ymm1 972; AVX2-NEXT: retq 973; 974; AVX512F-LABEL: mul_v64i8: 975; AVX512F: # %bb.0: # %entry 976; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31] 977; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] 978; AVX512F-NEXT: vpmullw %ymm4, %ymm5, %ymm4 979; AVX512F-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 980; AVX512F-NEXT: vpand %ymm5, %ymm4, %ymm4 981; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23] 982; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] 983; AVX512F-NEXT: vpmullw %ymm2, %ymm0, %ymm0 984; AVX512F-NEXT: vpand %ymm5, %ymm0, %ymm0 985; AVX512F-NEXT: vpackuswb %ymm4, %ymm0, %ymm0 986; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31] 987; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31] 988; AVX512F-NEXT: vpmullw %ymm2, %ymm4, %ymm2 989; AVX512F-NEXT: vpand %ymm5, %ymm2, %ymm2 990; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23] 991; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23] 992; AVX512F-NEXT: vpmullw %ymm3, %ymm1, %ymm1 993; AVX512F-NEXT: vpand %ymm5, %ymm1, %ymm1 994; AVX512F-NEXT: vpackuswb %ymm2, %ymm1, %ymm1 995; AVX512F-NEXT: retq 996; 997; AVX512BW-LABEL: mul_v64i8: 998; AVX512BW: # %bb.0: # %entry 999; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63] 1000; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] 1001; AVX512BW-NEXT: vpmullw %zmm2, %zmm3, %zmm2 1002; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] 1003; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm2 1004; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55] 1005; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55] 1006; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0 1007; AVX512BW-NEXT: vpandq %zmm3, %zmm0, %zmm0 1008; AVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0 1009; AVX512BW-NEXT: retq 1010entry: 1011 %A = mul <64 x i8> %i, %j 1012 ret <64 x i8> %A 1013} 1014 1015; PR30845 1016define <4 x i32> @mul_v4i64_zero_upper(<4 x i32> %val1, <4 x i32> %val2) { 1017; SSE2-LABEL: mul_v4i64_zero_upper: 1018; SSE2: # %bb.0: # %entry 1019; SSE2-NEXT: pxor %xmm3, %xmm3 1020; SSE2-NEXT: movdqa %xmm0, %xmm2 1021; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] 1022; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3] 1023; SSE2-NEXT: movdqa %xmm1, %xmm4 1024; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] 1025; SSE2-NEXT: pmuludq %xmm4, %xmm2 1026; SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3] 1027; SSE2-NEXT: pmuludq %xmm0, %xmm1 1028; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[1,3] 1029; SSE2-NEXT: movaps %xmm2, %xmm0 1030; SSE2-NEXT: retq 1031; 1032; SSE41-LABEL: mul_v4i64_zero_upper: 1033; SSE41: # %bb.0: # %entry 1034; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1] 1035; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero 1036; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero 1037; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] 1038; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero 1039; SSE41-NEXT: pmuludq %xmm2, %xmm4 1040; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero 1041; SSE41-NEXT: pmuludq %xmm3, %xmm0 1042; SSE41-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm4[1,3] 1043; SSE41-NEXT: retq 1044; 1045; AVX-LABEL: mul_v4i64_zero_upper: 1046; AVX: # %bb.0: # %entry 1047; AVX-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 1048; AVX-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 1049; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 1050; AVX-NEXT: vextracti128 $1, %ymm0, %xmm1 1051; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3] 1052; AVX-NEXT: vzeroupper 1053; AVX-NEXT: retq 1054entry: 1055 %val1a = zext <4 x i32> %val1 to <4 x i64> 1056 %val2a = zext <4 x i32> %val2 to <4 x i64> 1057 %res64 = mul <4 x i64> %val1a, %val2a 1058 %rescast = bitcast <4 x i64> %res64 to <8 x i32> 1059 %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 1060 ret <4 x i32> %res 1061} 1062 1063define <4 x i32> @mul_v4i64_zero_upper_left(<4 x i32> %val1, <4 x i64> %val2) { 1064; SSE2-LABEL: mul_v4i64_zero_upper_left: 1065; SSE2: # %bb.0: # %entry 1066; SSE2-NEXT: pxor %xmm3, %xmm3 1067; SSE2-NEXT: movdqa %xmm0, %xmm4 1068; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] 1069; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3] 1070; SSE2-NEXT: movdqa %xmm0, %xmm3 1071; SSE2-NEXT: pmuludq %xmm2, %xmm3 1072; SSE2-NEXT: psrlq $32, %xmm2 1073; SSE2-NEXT: pmuludq %xmm0, %xmm2 1074; SSE2-NEXT: psllq $32, %xmm2 1075; SSE2-NEXT: paddq %xmm3, %xmm2 1076; SSE2-NEXT: movdqa %xmm4, %xmm0 1077; SSE2-NEXT: pmuludq %xmm1, %xmm0 1078; SSE2-NEXT: psrlq $32, %xmm1 1079; SSE2-NEXT: pmuludq %xmm4, %xmm1 1080; SSE2-NEXT: psllq $32, %xmm1 1081; SSE2-NEXT: paddq %xmm1, %xmm0 1082; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3] 1083; SSE2-NEXT: retq 1084; 1085; SSE41-LABEL: mul_v4i64_zero_upper_left: 1086; SSE41: # %bb.0: # %entry 1087; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1] 1088; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero 1089; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero 1090; SSE41-NEXT: movdqa %xmm4, %xmm0 1091; SSE41-NEXT: pmuludq %xmm1, %xmm0 1092; SSE41-NEXT: psrlq $32, %xmm1 1093; SSE41-NEXT: pmuludq %xmm4, %xmm1 1094; SSE41-NEXT: psllq $32, %xmm1 1095; SSE41-NEXT: paddq %xmm1, %xmm0 1096; SSE41-NEXT: movdqa %xmm3, %xmm1 1097; SSE41-NEXT: pmuludq %xmm2, %xmm1 1098; SSE41-NEXT: psrlq $32, %xmm2 1099; SSE41-NEXT: pmuludq %xmm3, %xmm2 1100; SSE41-NEXT: psllq $32, %xmm2 1101; SSE41-NEXT: paddq %xmm1, %xmm2 1102; SSE41-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3] 1103; SSE41-NEXT: retq 1104; 1105; AVX-LABEL: mul_v4i64_zero_upper_left: 1106; AVX: # %bb.0: # %entry 1107; AVX-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 1108; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm2 1109; AVX-NEXT: vpsrlq $32, %ymm1, %ymm1 1110; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 1111; AVX-NEXT: vpsllq $32, %ymm0, %ymm0 1112; AVX-NEXT: vpaddq %ymm0, %ymm2, %ymm0 1113; AVX-NEXT: vextracti128 $1, %ymm0, %xmm1 1114; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3] 1115; AVX-NEXT: vzeroupper 1116; AVX-NEXT: retq 1117entry: 1118 %val1a = zext <4 x i32> %val1 to <4 x i64> 1119 %res64 = mul <4 x i64> %val1a, %val2 1120 %rescast = bitcast <4 x i64> %res64 to <8 x i32> 1121 %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 1122 ret <4 x i32> %res 1123} 1124 1125define <4 x i32> @mul_v4i64_zero_lower(<4 x i32> %val1, <4 x i64> %val2) { 1126; SSE2-LABEL: mul_v4i64_zero_lower: 1127; SSE2: # %bb.0: # %entry 1128; SSE2-NEXT: pxor %xmm4, %xmm4 1129; SSE2-NEXT: movdqa %xmm0, %xmm3 1130; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1] 1131; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm4[2],xmm0[3],xmm4[3] 1132; SSE2-NEXT: psrlq $32, %xmm2 1133; SSE2-NEXT: pmuludq %xmm0, %xmm2 1134; SSE2-NEXT: psrlq $32, %xmm1 1135; SSE2-NEXT: pmuludq %xmm1, %xmm3 1136; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm2[0,2] 1137; SSE2-NEXT: movaps %xmm3, %xmm0 1138; SSE2-NEXT: retq 1139; 1140; SSE41-LABEL: mul_v4i64_zero_lower: 1141; SSE41: # %bb.0: # %entry 1142; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1] 1143; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero 1144; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero 1145; SSE41-NEXT: psrlq $32, %xmm1 1146; SSE41-NEXT: pmuludq %xmm1, %xmm0 1147; SSE41-NEXT: psrlq $32, %xmm2 1148; SSE41-NEXT: pmuludq %xmm3, %xmm2 1149; SSE41-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2] 1150; SSE41-NEXT: retq 1151; 1152; AVX-LABEL: mul_v4i64_zero_lower: 1153; AVX: # %bb.0: # %entry 1154; AVX-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 1155; AVX-NEXT: vpsrlq $32, %ymm1, %ymm1 1156; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 1157; AVX-NEXT: vpsllq $32, %ymm0, %ymm0 1158; AVX-NEXT: vextracti128 $1, %ymm0, %xmm1 1159; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3] 1160; AVX-NEXT: vzeroupper 1161; AVX-NEXT: retq 1162entry: 1163 %val1a = zext <4 x i32> %val1 to <4 x i64> 1164 %val2a = and <4 x i64> %val2, <i64 -4294967296, i64 -4294967296, i64 -4294967296, i64 -4294967296> 1165 %res64 = mul <4 x i64> %val1a, %val2a 1166 %rescast = bitcast <4 x i64> %res64 to <8 x i32> 1167 %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> 1168 ret <4 x i32> %res 1169} 1170 1171define <8 x i32> @mul_v8i64_zero_upper(<8 x i32> %val1, <8 x i32> %val2) { 1172; SSE2-LABEL: mul_v8i64_zero_upper: 1173; SSE2: # %bb.0: # %entry 1174; SSE2-NEXT: pxor %xmm6, %xmm6 1175; SSE2-NEXT: movdqa %xmm0, %xmm4 1176; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1] 1177; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm6[2],xmm0[3],xmm6[3] 1178; SSE2-NEXT: movdqa %xmm1, %xmm5 1179; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1] 1180; SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm6[2],xmm1[3],xmm6[3] 1181; SSE2-NEXT: movdqa %xmm2, %xmm7 1182; SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1] 1183; SSE2-NEXT: pmuludq %xmm7, %xmm4 1184; SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm6[2],xmm2[3],xmm6[3] 1185; SSE2-NEXT: pmuludq %xmm0, %xmm2 1186; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,3],xmm2[1,3] 1187; SSE2-NEXT: movdqa %xmm3, %xmm0 1188; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1] 1189; SSE2-NEXT: pmuludq %xmm0, %xmm5 1190; SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm6[2],xmm3[3],xmm6[3] 1191; SSE2-NEXT: pmuludq %xmm1, %xmm3 1192; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,3],xmm3[1,3] 1193; SSE2-NEXT: movaps %xmm4, %xmm0 1194; SSE2-NEXT: movaps %xmm5, %xmm1 1195; SSE2-NEXT: retq 1196; 1197; SSE41-LABEL: mul_v8i64_zero_upper: 1198; SSE41: # %bb.0: # %entry 1199; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1] 1200; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero 1201; SSE41-NEXT: pmovzxdq {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero 1202; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] 1203; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero 1204; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = xmm1[0],zero,xmm1[1],zero 1205; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1] 1206; SSE41-NEXT: pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero 1207; SSE41-NEXT: pmuludq %xmm4, %xmm1 1208; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero 1209; SSE41-NEXT: pmuludq %xmm5, %xmm0 1210; SSE41-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3] 1211; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1] 1212; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero 1213; SSE41-NEXT: pmuludq %xmm6, %xmm2 1214; SSE41-NEXT: pmovzxdq {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero 1215; SSE41-NEXT: pmuludq %xmm7, %xmm1 1216; SSE41-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,3],xmm2[1,3] 1217; SSE41-NEXT: retq 1218; 1219; AVX2-LABEL: mul_v8i64_zero_upper: 1220; AVX2: # %bb.0: # %entry 1221; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 1222; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 1223; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 1224; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 1225; AVX2-NEXT: vpmuludq %ymm3, %ymm2, %ymm2 1226; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1 1227; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 1228; AVX2-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 1229; AVX2-NEXT: vshufps {{.*#+}} ymm0 = ymm2[1,3],ymm0[1,3],ymm2[5,7],ymm0[5,7] 1230; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3] 1231; AVX2-NEXT: retq 1232; 1233; AVX512-LABEL: mul_v8i64_zero_upper: 1234; AVX512: # %bb.0: # %entry 1235; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero 1236; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero 1237; AVX512-NEXT: vpmuludq %zmm1, %zmm0, %zmm0 1238; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 1239; AVX512-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7] 1240; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3] 1241; AVX512-NEXT: retq 1242entry: 1243 %val1a = zext <8 x i32> %val1 to <8 x i64> 1244 %val2a = zext <8 x i32> %val2 to <8 x i64> 1245 %res64 = mul <8 x i64> %val1a, %val2a 1246 %rescast = bitcast <8 x i64> %res64 to <16 x i32> 1247 %res = shufflevector <16 x i32> %rescast, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7,i32 9, i32 11, i32 13, i32 15 > 1248 ret <8 x i32> %res 1249} 1250 1251define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) { 1252; SSE2-LABEL: mul_v8i64_sext: 1253; SSE2: # %bb.0: 1254; SSE2-NEXT: movdqa %xmm1, %xmm12 1255; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm0[4],xmm10[5],xmm0[5],xmm10[6],xmm0[6],xmm10[7],xmm0[7] 1256; SSE2-NEXT: psrad $16, %xmm10 1257; SSE2-NEXT: pxor %xmm8, %xmm8 1258; SSE2-NEXT: pxor %xmm13, %xmm13 1259; SSE2-NEXT: pcmpgtd %xmm10, %xmm13 1260; SSE2-NEXT: movdqa %xmm10, %xmm9 1261; SSE2-NEXT: punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm13[2],xmm9[3],xmm13[3] 1262; SSE2-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm13[0],xmm10[1],xmm13[1] 1263; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] 1264; SSE2-NEXT: psrad $16, %xmm0 1265; SSE2-NEXT: pxor %xmm15, %xmm15 1266; SSE2-NEXT: pcmpgtd %xmm0, %xmm15 1267; SSE2-NEXT: movdqa %xmm0, %xmm11 1268; SSE2-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm15[2],xmm11[3],xmm15[3] 1269; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1] 1270; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1] 1271; SSE2-NEXT: pxor %xmm14, %xmm14 1272; SSE2-NEXT: pcmpgtd %xmm3, %xmm14 1273; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm14[0],xmm3[1],xmm14[1] 1274; SSE2-NEXT: pxor %xmm5, %xmm5 1275; SSE2-NEXT: pcmpgtd %xmm2, %xmm5 1276; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1] 1277; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] 1278; SSE2-NEXT: pxor %xmm6, %xmm6 1279; SSE2-NEXT: pcmpgtd %xmm1, %xmm6 1280; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1] 1281; SSE2-NEXT: pxor %xmm7, %xmm7 1282; SSE2-NEXT: pcmpgtd %xmm12, %xmm7 1283; SSE2-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm7[0],xmm12[1],xmm7[1] 1284; SSE2-NEXT: movdqa %xmm15, %xmm4 1285; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1] 1286; SSE2-NEXT: pmuludq %xmm12, %xmm4 1287; SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1] 1288; SSE2-NEXT: pmuludq %xmm0, %xmm7 1289; SSE2-NEXT: paddq %xmm4, %xmm7 1290; SSE2-NEXT: psllq $32, %xmm7 1291; SSE2-NEXT: pmuludq %xmm12, %xmm0 1292; SSE2-NEXT: paddq %xmm7, %xmm0 1293; SSE2-NEXT: punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm8[2],xmm15[3],xmm8[3] 1294; SSE2-NEXT: pmuludq %xmm1, %xmm15 1295; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1] 1296; SSE2-NEXT: pmuludq %xmm11, %xmm6 1297; SSE2-NEXT: paddq %xmm15, %xmm6 1298; SSE2-NEXT: psllq $32, %xmm6 1299; SSE2-NEXT: pmuludq %xmm11, %xmm1 1300; SSE2-NEXT: paddq %xmm6, %xmm1 1301; SSE2-NEXT: movdqa %xmm13, %xmm4 1302; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1] 1303; SSE2-NEXT: pmuludq %xmm2, %xmm4 1304; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1] 1305; SSE2-NEXT: pmuludq %xmm10, %xmm5 1306; SSE2-NEXT: paddq %xmm4, %xmm5 1307; SSE2-NEXT: psllq $32, %xmm5 1308; SSE2-NEXT: pmuludq %xmm10, %xmm2 1309; SSE2-NEXT: paddq %xmm5, %xmm2 1310; SSE2-NEXT: punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm8[2],xmm13[3],xmm8[3] 1311; SSE2-NEXT: pmuludq %xmm3, %xmm13 1312; SSE2-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm8[0],xmm14[1],xmm8[1] 1313; SSE2-NEXT: pmuludq %xmm9, %xmm14 1314; SSE2-NEXT: paddq %xmm13, %xmm14 1315; SSE2-NEXT: psllq $32, %xmm14 1316; SSE2-NEXT: pmuludq %xmm9, %xmm3 1317; SSE2-NEXT: paddq %xmm14, %xmm3 1318; SSE2-NEXT: retq 1319; 1320; SSE41-LABEL: mul_v8i64_sext: 1321; SSE41: # %bb.0: 1322; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3] 1323; SSE41-NEXT: pmovsxwq %xmm3, %xmm4 1324; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1] 1325; SSE41-NEXT: pmovsxwq %xmm3, %xmm5 1326; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3] 1327; SSE41-NEXT: pmovsxwq %xmm3, %xmm6 1328; SSE41-NEXT: pmovsxwq %xmm0, %xmm7 1329; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1] 1330; SSE41-NEXT: pmovsxdq %xmm0, %xmm3 1331; SSE41-NEXT: pmuldq %xmm4, %xmm3 1332; SSE41-NEXT: pmovsxdq %xmm2, %xmm2 1333; SSE41-NEXT: pmuldq %xmm5, %xmm2 1334; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] 1335; SSE41-NEXT: pmovsxdq %xmm0, %xmm4 1336; SSE41-NEXT: pmuldq %xmm6, %xmm4 1337; SSE41-NEXT: pmovsxdq %xmm1, %xmm0 1338; SSE41-NEXT: pmuldq %xmm7, %xmm0 1339; SSE41-NEXT: movdqa %xmm4, %xmm1 1340; SSE41-NEXT: retq 1341; 1342; AVX2-LABEL: mul_v8i64_sext: 1343; AVX2: # %bb.0: 1344; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1] 1345; AVX2-NEXT: vpmovsxwq %xmm2, %ymm2 1346; AVX2-NEXT: vpmovsxwq %xmm0, %ymm0 1347; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3 1348; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero 1349; AVX2-NEXT: vpmuldq %ymm3, %ymm2, %ymm2 1350; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 1351; AVX2-NEXT: vpmuldq %ymm1, %ymm0, %ymm0 1352; AVX2-NEXT: vmovdqa %ymm2, %ymm1 1353; AVX2-NEXT: retq 1354; 1355; AVX512-LABEL: mul_v8i64_sext: 1356; AVX512: # %bb.0: 1357; AVX512-NEXT: vpmovsxwq %xmm0, %zmm0 1358; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero 1359; AVX512-NEXT: vpmuldq %zmm1, %zmm0, %zmm0 1360; AVX512-NEXT: retq 1361 %1 = sext <8 x i16> %val1 to <8 x i64> 1362 %2 = sext <8 x i32> %val2 to <8 x i64> 1363 %3 = mul <8 x i64> %1, %2 1364 ret <8 x i64> %3 1365} 1366 1367define <2 x i64> @pmuldq_square(<2 x i64> %x) { 1368; SSE2-LABEL: pmuldq_square: 1369; SSE2: # %bb.0: 1370; SSE2-NEXT: psllq $32, %xmm0 1371; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3] 1372; SSE2-NEXT: psrad $31, %xmm0 1373; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3] 1374; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] 1375; SSE2-NEXT: psrlq $32, %xmm0 1376; SSE2-NEXT: pmuludq %xmm1, %xmm0 1377; SSE2-NEXT: paddq %xmm0, %xmm0 1378; SSE2-NEXT: psllq $32, %xmm0 1379; SSE2-NEXT: pmuludq %xmm1, %xmm1 1380; SSE2-NEXT: paddq %xmm1, %xmm0 1381; SSE2-NEXT: retq 1382; 1383; SSE41-LABEL: pmuldq_square: 1384; SSE41: # %bb.0: 1385; SSE41-NEXT: pmuldq %xmm0, %xmm0 1386; SSE41-NEXT: retq 1387; 1388; AVX-LABEL: pmuldq_square: 1389; AVX: # %bb.0: 1390; AVX-NEXT: vpmuldq %xmm0, %xmm0, %xmm0 1391; AVX-NEXT: retq 1392 %1 = shl <2 x i64> %x, <i64 32, i64 32> 1393 %2 = ashr exact <2 x i64> %1, <i64 32, i64 32> 1394 %3 = mul nsw <2 x i64> %2, %2 1395 ret <2 x i64> %3 1396} 1397 1398define <2 x i64> @pmuludq_square(<2 x i64> %x) { 1399; SSE-LABEL: pmuludq_square: 1400; SSE: # %bb.0: 1401; SSE-NEXT: pmuludq %xmm0, %xmm0 1402; SSE-NEXT: retq 1403; 1404; AVX-LABEL: pmuludq_square: 1405; AVX: # %bb.0: 1406; AVX-NEXT: vpmuludq %xmm0, %xmm0, %xmm0 1407; AVX-NEXT: retq 1408 %1 = and <2 x i64> %x, <i64 4294967295, i64 4294967295> 1409 %2 = mul nuw <2 x i64> %1, %1 1410 ret <2 x i64> %2 1411} 1412