1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK 3 4declare {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32>, <1 x i32>) 5declare {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32>, <2 x i32>) 6declare {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32>, <3 x i32>) 7declare {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32>, <4 x i32>) 8declare {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32>, <6 x i32>) 9declare {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>) 10 11declare {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8>, <16 x i8>) 12declare {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16>, <8 x i16>) 13declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>) 14 15declare {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24>, <4 x i24>) 16declare {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1>, <4 x i1>) 17declare {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128>, <2 x i128>) 18 19define <1 x i32> @umulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind { 20; CHECK-LABEL: umulo_v1i32: 21; CHECK: // %bb.0: 22; CHECK-NEXT: umull v1.2d, v0.2s, v1.2s 23; CHECK-NEXT: shrn v0.2s, v1.2d, #32 24; CHECK-NEXT: xtn v1.2s, v1.2d 25; CHECK-NEXT: cmtst v0.2s, v0.2s, v0.2s 26; CHECK-NEXT: str s1, [x0] 27; CHECK-NEXT: ret 28 %t = call {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1) 29 %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0 30 %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1 31 %res = sext <1 x i1> %obit to <1 x i32> 32 store <1 x i32> %val, <1 x i32>* %p2 33 ret <1 x i32> %res 34} 35 36define <2 x i32> @umulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind { 37; CHECK-LABEL: umulo_v2i32: 38; CHECK: // %bb.0: 39; CHECK-NEXT: umull v1.2d, v0.2s, v1.2s 40; CHECK-NEXT: shrn v0.2s, v1.2d, #32 41; CHECK-NEXT: xtn v1.2s, v1.2d 42; CHECK-NEXT: cmtst v0.2s, v0.2s, v0.2s 43; CHECK-NEXT: str d1, [x0] 44; CHECK-NEXT: ret 45 %t = call {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1) 46 %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0 47 %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1 48 %res = sext <2 x i1> %obit to <2 x i32> 49 store <2 x i32> %val, <2 x i32>* %p2 50 ret <2 x i32> %res 51} 52 53define <3 x i32> @umulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind { 54; CHECK-LABEL: umulo_v3i32: 55; CHECK: // %bb.0: 56; CHECK-NEXT: umull2 v2.2d, v0.4s, v1.4s 57; CHECK-NEXT: add x8, x0, #8 58; CHECK-NEXT: umull v3.2d, v0.2s, v1.2s 59; CHECK-NEXT: mul v1.4s, v0.4s, v1.4s 60; CHECK-NEXT: uzp2 v2.4s, v3.4s, v2.4s 61; CHECK-NEXT: st1 { v1.s }[2], [x8] 62; CHECK-NEXT: str d1, [x0] 63; CHECK-NEXT: cmtst v2.4s, v2.4s, v2.4s 64; CHECK-NEXT: mov v0.16b, v2.16b 65; CHECK-NEXT: ret 66 %t = call {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1) 67 %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0 68 %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1 69 %res = sext <3 x i1> %obit to <3 x i32> 70 store <3 x i32> %val, <3 x i32>* %p2 71 ret <3 x i32> %res 72} 73 74define <4 x i32> @umulo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind { 75; CHECK-LABEL: umulo_v4i32: 76; CHECK: // %bb.0: 77; CHECK-NEXT: umull2 v2.2d, v0.4s, v1.4s 78; CHECK-NEXT: umull v3.2d, v0.2s, v1.2s 79; CHECK-NEXT: mul v1.4s, v0.4s, v1.4s 80; CHECK-NEXT: uzp2 v2.4s, v3.4s, v2.4s 81; CHECK-NEXT: str q1, [x0] 82; CHECK-NEXT: cmtst v2.4s, v2.4s, v2.4s 83; CHECK-NEXT: mov v0.16b, v2.16b 84; CHECK-NEXT: ret 85 %t = call {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1) 86 %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0 87 %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1 88 %res = sext <4 x i1> %obit to <4 x i32> 89 store <4 x i32> %val, <4 x i32>* %p2 90 ret <4 x i32> %res 91} 92 93define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind { 94; CHECK-LABEL: umulo_v6i32: 95; CHECK: // %bb.0: 96; CHECK-NEXT: fmov s0, w6 97; CHECK-NEXT: fmov s1, w0 98; CHECK-NEXT: mov x8, sp 99; CHECK-NEXT: ldr s2, [sp, #16] 100; CHECK-NEXT: add x9, sp, #24 101; CHECK-NEXT: add x10, sp, #8 102; CHECK-NEXT: mov v0.s[1], w7 103; CHECK-NEXT: fmov s3, w4 104; CHECK-NEXT: mov v1.s[1], w1 105; CHECK-NEXT: ld1 { v2.s }[1], [x9] 106; CHECK-NEXT: mov v3.s[1], w5 107; CHECK-NEXT: ld1 { v0.s }[2], [x8] 108; CHECK-NEXT: mov v1.s[2], w2 109; CHECK-NEXT: ldr x8, [sp, #32] 110; CHECK-NEXT: umull2 v4.2d, v3.4s, v2.4s 111; CHECK-NEXT: ld1 { v0.s }[3], [x10] 112; CHECK-NEXT: mov v1.s[3], w3 113; CHECK-NEXT: umull v7.2d, v3.2s, v2.2s 114; CHECK-NEXT: mul v2.4s, v3.4s, v2.4s 115; CHECK-NEXT: umull2 v5.2d, v1.4s, v0.4s 116; CHECK-NEXT: umull v6.2d, v1.2s, v0.2s 117; CHECK-NEXT: uzp2 v4.4s, v7.4s, v4.4s 118; CHECK-NEXT: str d2, [x8, #16] 119; CHECK-NEXT: mul v0.4s, v1.4s, v0.4s 120; CHECK-NEXT: uzp2 v5.4s, v6.4s, v5.4s 121; CHECK-NEXT: cmtst v4.4s, v4.4s, v4.4s 122; CHECK-NEXT: str q0, [x8] 123; CHECK-NEXT: cmtst v3.4s, v5.4s, v5.4s 124; CHECK-NEXT: mov w5, v4.s[1] 125; CHECK-NEXT: fmov w4, s4 126; CHECK-NEXT: mov w1, v3.s[1] 127; CHECK-NEXT: mov w2, v3.s[2] 128; CHECK-NEXT: mov w3, v3.s[3] 129; CHECK-NEXT: fmov w0, s3 130; CHECK-NEXT: ret 131 %t = call {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1) 132 %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0 133 %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1 134 %res = sext <6 x i1> %obit to <6 x i32> 135 store <6 x i32> %val, <6 x i32>* %p2 136 ret <6 x i32> %res 137} 138 139define <8 x i32> @umulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind { 140; CHECK-LABEL: umulo_v8i32: 141; CHECK: // %bb.0: 142; CHECK-NEXT: umull2 v4.2d, v1.4s, v3.4s 143; CHECK-NEXT: umull2 v5.2d, v0.4s, v2.4s 144; CHECK-NEXT: umull v6.2d, v0.2s, v2.2s 145; CHECK-NEXT: umull v7.2d, v1.2s, v3.2s 146; CHECK-NEXT: mul v3.4s, v1.4s, v3.4s 147; CHECK-NEXT: mul v2.4s, v0.4s, v2.4s 148; CHECK-NEXT: uzp2 v5.4s, v6.4s, v5.4s 149; CHECK-NEXT: uzp2 v6.4s, v7.4s, v4.4s 150; CHECK-NEXT: stp q2, q3, [x0] 151; CHECK-NEXT: cmtst v4.4s, v5.4s, v5.4s 152; CHECK-NEXT: cmtst v5.4s, v6.4s, v6.4s 153; CHECK-NEXT: mov v0.16b, v4.16b 154; CHECK-NEXT: mov v1.16b, v5.16b 155; CHECK-NEXT: ret 156 %t = call {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1) 157 %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0 158 %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1 159 %res = sext <8 x i1> %obit to <8 x i32> 160 store <8 x i32> %val, <8 x i32>* %p2 161 ret <8 x i32> %res 162} 163 164define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind { 165; CHECK-LABEL: umulo_v16i8: 166; CHECK: // %bb.0: 167; CHECK-NEXT: umull2 v2.8h, v0.16b, v1.16b 168; CHECK-NEXT: umull v3.8h, v0.8b, v1.8b 169; CHECK-NEXT: mul v5.16b, v0.16b, v1.16b 170; CHECK-NEXT: uzp2 v2.16b, v3.16b, v2.16b 171; CHECK-NEXT: str q5, [x0] 172; CHECK-NEXT: cmtst v2.16b, v2.16b, v2.16b 173; CHECK-NEXT: zip1 v3.8b, v2.8b, v0.8b 174; CHECK-NEXT: zip2 v4.8b, v2.8b, v0.8b 175; CHECK-NEXT: ext v0.16b, v2.16b, v2.16b, #8 176; CHECK-NEXT: ushll v1.4s, v3.4h, #0 177; CHECK-NEXT: ushll v2.4s, v4.4h, #0 178; CHECK-NEXT: zip1 v3.8b, v0.8b, v0.8b 179; CHECK-NEXT: zip2 v4.8b, v0.8b, v0.8b 180; CHECK-NEXT: shl v1.4s, v1.4s, #31 181; CHECK-NEXT: shl v2.4s, v2.4s, #31 182; CHECK-NEXT: cmlt v0.4s, v1.4s, #0 183; CHECK-NEXT: cmlt v1.4s, v2.4s, #0 184; CHECK-NEXT: ushll v2.4s, v3.4h, #0 185; CHECK-NEXT: ushll v3.4s, v4.4h, #0 186; CHECK-NEXT: shl v2.4s, v2.4s, #31 187; CHECK-NEXT: shl v3.4s, v3.4s, #31 188; CHECK-NEXT: cmlt v2.4s, v2.4s, #0 189; CHECK-NEXT: cmlt v3.4s, v3.4s, #0 190; CHECK-NEXT: ret 191 %t = call {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1) 192 %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0 193 %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1 194 %res = sext <16 x i1> %obit to <16 x i32> 195 store <16 x i8> %val, <16 x i8>* %p2 196 ret <16 x i32> %res 197} 198 199define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind { 200; CHECK-LABEL: umulo_v8i16: 201; CHECK: // %bb.0: 202; CHECK-NEXT: umull2 v2.4s, v0.8h, v1.8h 203; CHECK-NEXT: umull v3.4s, v0.4h, v1.4h 204; CHECK-NEXT: uzp2 v2.8h, v3.8h, v2.8h 205; CHECK-NEXT: cmtst v2.8h, v2.8h, v2.8h 206; CHECK-NEXT: xtn v2.8b, v2.8h 207; CHECK-NEXT: zip1 v3.8b, v2.8b, v0.8b 208; CHECK-NEXT: zip2 v2.8b, v2.8b, v0.8b 209; CHECK-NEXT: ushll v3.4s, v3.4h, #0 210; CHECK-NEXT: ushll v2.4s, v2.4h, #0 211; CHECK-NEXT: shl v3.4s, v3.4s, #31 212; CHECK-NEXT: shl v4.4s, v2.4s, #31 213; CHECK-NEXT: cmlt v2.4s, v3.4s, #0 214; CHECK-NEXT: cmlt v3.4s, v4.4s, #0 215; CHECK-NEXT: mul v4.8h, v0.8h, v1.8h 216; CHECK-NEXT: mov v0.16b, v2.16b 217; CHECK-NEXT: mov v1.16b, v3.16b 218; CHECK-NEXT: str q4, [x0] 219; CHECK-NEXT: ret 220 %t = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1) 221 %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0 222 %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1 223 %res = sext <8 x i1> %obit to <8 x i32> 224 store <8 x i16> %val, <8 x i16>* %p2 225 ret <8 x i32> %res 226} 227 228define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind { 229; CHECK-LABEL: umulo_v2i64: 230; CHECK: // %bb.0: 231; CHECK-NEXT: mov x8, v1.d[1] 232; CHECK-NEXT: fmov x10, d1 233; CHECK-NEXT: mov x9, v0.d[1] 234; CHECK-NEXT: fmov x11, d0 235; CHECK-NEXT: umulh x12, x9, x8 236; CHECK-NEXT: umulh x13, x11, x10 237; CHECK-NEXT: cmp xzr, x12 238; CHECK-NEXT: mul x10, x11, x10 239; CHECK-NEXT: csetm x12, ne 240; CHECK-NEXT: cmp xzr, x13 241; CHECK-NEXT: csetm x13, ne 242; CHECK-NEXT: mul x8, x9, x8 243; CHECK-NEXT: fmov d1, x10 244; CHECK-NEXT: fmov d0, x13 245; CHECK-NEXT: mov v1.d[1], x8 246; CHECK-NEXT: mov v0.d[1], x12 247; CHECK-NEXT: str q1, [x0] 248; CHECK-NEXT: xtn v0.2s, v0.2d 249; CHECK-NEXT: ret 250 %t = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1) 251 %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0 252 %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1 253 %res = sext <2 x i1> %obit to <2 x i32> 254 store <2 x i64> %val, <2 x i64>* %p2 255 ret <2 x i32> %res 256} 257 258define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind { 259; CHECK-LABEL: umulo_v4i24: 260; CHECK: // %bb.0: 261; CHECK-NEXT: bic v1.4s, #255, lsl #24 262; CHECK-NEXT: bic v0.4s, #255, lsl #24 263; CHECK-NEXT: mul v2.4s, v0.4s, v1.4s 264; CHECK-NEXT: umull2 v3.2d, v0.4s, v1.4s 265; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s 266; CHECK-NEXT: mov w8, v2.s[3] 267; CHECK-NEXT: mov w10, v2.s[2] 268; CHECK-NEXT: mov w11, v2.s[1] 269; CHECK-NEXT: ushr v1.4s, v2.4s, #24 270; CHECK-NEXT: uzp2 v0.4s, v0.4s, v3.4s 271; CHECK-NEXT: fmov w9, s2 272; CHECK-NEXT: cmtst v1.4s, v1.4s, v1.4s 273; CHECK-NEXT: sturh w8, [x0, #9] 274; CHECK-NEXT: lsr w8, w8, #16 275; CHECK-NEXT: strh w10, [x0, #6] 276; CHECK-NEXT: lsr w10, w10, #16 277; CHECK-NEXT: cmeq v0.4s, v0.4s, #0 278; CHECK-NEXT: sturh w11, [x0, #3] 279; CHECK-NEXT: lsr w11, w11, #16 280; CHECK-NEXT: strb w8, [x0, #11] 281; CHECK-NEXT: lsr w8, w9, #16 282; CHECK-NEXT: strh w9, [x0] 283; CHECK-NEXT: orn v0.16b, v1.16b, v0.16b 284; CHECK-NEXT: strb w10, [x0, #8] 285; CHECK-NEXT: strb w11, [x0, #5] 286; CHECK-NEXT: strb w8, [x0, #2] 287; CHECK-NEXT: ret 288 %t = call {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1) 289 %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0 290 %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1 291 %res = sext <4 x i1> %obit to <4 x i32> 292 store <4 x i24> %val, <4 x i24>* %p2 293 ret <4 x i32> %res 294} 295 296define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind { 297; CHECK-LABEL: umulo_v4i1: 298; CHECK: // %bb.0: 299; CHECK-NEXT: fmov d2, d0 300; CHECK-NEXT: movi v0.2d, #0000000000000000 301; CHECK-NEXT: and v1.8b, v2.8b, v1.8b 302; CHECK-NEXT: umov w8, v1.h[1] 303; CHECK-NEXT: umov w9, v1.h[2] 304; CHECK-NEXT: umov w10, v1.h[0] 305; CHECK-NEXT: umov w11, v1.h[3] 306; CHECK-NEXT: and w8, w8, #0x1 307; CHECK-NEXT: and w9, w9, #0x1 308; CHECK-NEXT: bfi w10, w8, #1, #1 309; CHECK-NEXT: bfi w10, w9, #2, #1 310; CHECK-NEXT: bfi w10, w11, #3, #29 311; CHECK-NEXT: and w8, w10, #0xf 312; CHECK-NEXT: strb w8, [x0] 313; CHECK-NEXT: ret 314 %t = call {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1) 315 %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0 316 %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1 317 %res = sext <4 x i1> %obit to <4 x i32> 318 store <4 x i1> %val, <4 x i1>* %p2 319 ret <4 x i32> %res 320} 321 322define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind { 323; CHECK-LABEL: umulo_v2i128: 324; CHECK: // %bb.0: 325; CHECK-NEXT: mul x8, x7, x2 326; CHECK-NEXT: umulh x9, x2, x6 327; CHECK-NEXT: madd x8, x3, x6, x8 328; CHECK-NEXT: umulh x10, x3, x6 329; CHECK-NEXT: adds x8, x9, x8 330; CHECK-NEXT: umulh x11, x7, x2 331; CHECK-NEXT: cset w9, hs 332; CHECK-NEXT: cmp x3, #0 333; CHECK-NEXT: ccmp x7, #0, #4, ne 334; CHECK-NEXT: umulh x13, x1, x4 335; CHECK-NEXT: ccmp xzr, x10, #0, eq 336; CHECK-NEXT: mul x10, x5, x0 337; CHECK-NEXT: madd x10, x1, x4, x10 338; CHECK-NEXT: ccmp xzr, x11, #0, eq 339; CHECK-NEXT: umulh x11, x0, x4 340; CHECK-NEXT: cset w12, ne 341; CHECK-NEXT: adds x10, x11, x10 342; CHECK-NEXT: cset w11, hs 343; CHECK-NEXT: cmp x1, #0 344; CHECK-NEXT: ccmp x5, #0, #4, ne 345; CHECK-NEXT: orr w9, w12, w9 346; CHECK-NEXT: mul x12, x0, x4 347; CHECK-NEXT: ccmp xzr, x13, #0, eq 348; CHECK-NEXT: umulh x13, x5, x0 349; CHECK-NEXT: ccmp xzr, x13, #0, eq 350; CHECK-NEXT: cset w13, ne 351; CHECK-NEXT: orr w11, w13, w11 352; CHECK-NEXT: fmov s0, w11 353; CHECK-NEXT: ldr x11, [sp] 354; CHECK-NEXT: mov v0.s[1], w9 355; CHECK-NEXT: mul x9, x2, x6 356; CHECK-NEXT: stp x12, x10, [x11] 357; CHECK-NEXT: shl v0.2s, v0.2s, #31 358; CHECK-NEXT: stp x9, x8, [x11, #16] 359; CHECK-NEXT: cmlt v0.2s, v0.2s, #0 360; CHECK-NEXT: ret 361 %t = call {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1) 362 %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0 363 %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1 364 %res = sext <2 x i1> %obit to <2 x i32> 365 store <2 x i128> %val, <2 x i128>* %p2 366 ret <2 x i32> %res 367} 368