1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BASE 3; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+dotprod %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DOT 4 5define i32 @add_v4i32_v4i32(<4 x i32> %x) { 6; CHECK-LABEL: add_v4i32_v4i32: 7; CHECK: // %bb.0: // %entry 8; CHECK-NEXT: addv s0, v0.4s 9; CHECK-NEXT: fmov w0, s0 10; CHECK-NEXT: ret 11entry: 12 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x) 13 ret i32 %z 14} 15 16define i64 @add_v4i32_v4i64_zext(<4 x i32> %x) { 17; CHECK-LABEL: add_v4i32_v4i64_zext: 18; CHECK: // %bb.0: // %entry 19; CHECK-NEXT: uaddlv d0, v0.4s 20; CHECK-NEXT: fmov x0, d0 21; CHECK-NEXT: ret 22entry: 23 %xx = zext <4 x i32> %x to <4 x i64> 24 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 25 ret i64 %z 26} 27 28define i64 @add_v4i32_v4i64_sext(<4 x i32> %x) { 29; CHECK-LABEL: add_v4i32_v4i64_sext: 30; CHECK: // %bb.0: // %entry 31; CHECK-NEXT: saddlv d0, v0.4s 32; CHECK-NEXT: fmov x0, d0 33; CHECK-NEXT: ret 34entry: 35 %xx = sext <4 x i32> %x to <4 x i64> 36 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 37 ret i64 %z 38} 39 40define i64 @add_v2i32_v2i64_zext(<2 x i32> %x) { 41; CHECK-LABEL: add_v2i32_v2i64_zext: 42; CHECK: // %bb.0: // %entry 43; CHECK-NEXT: ushll v0.2d, v0.2s, #0 44; CHECK-NEXT: addp d0, v0.2d 45; CHECK-NEXT: fmov x0, d0 46; CHECK-NEXT: ret 47entry: 48 %xx = zext <2 x i32> %x to <2 x i64> 49 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 50 ret i64 %z 51} 52 53define i64 @add_v2i32_v2i64_sext(<2 x i32> %x) { 54; CHECK-LABEL: add_v2i32_v2i64_sext: 55; CHECK: // %bb.0: // %entry 56; CHECK-NEXT: sshll v0.2d, v0.2s, #0 57; CHECK-NEXT: addp d0, v0.2d 58; CHECK-NEXT: fmov x0, d0 59; CHECK-NEXT: ret 60entry: 61 %xx = sext <2 x i32> %x to <2 x i64> 62 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 63 ret i64 %z 64} 65 66define i32 @add_v8i16_v8i32_zext(<8 x i16> %x) { 67; CHECK-LABEL: add_v8i16_v8i32_zext: 68; CHECK: // %bb.0: // %entry 69; CHECK-NEXT: uaddlv s0, v0.8h 70; CHECK-NEXT: fmov w0, s0 71; CHECK-NEXT: ret 72entry: 73 %xx = zext <8 x i16> %x to <8 x i32> 74 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) 75 ret i32 %z 76} 77 78define i32 @add_v8i16_v8i32_sext(<8 x i16> %x) { 79; CHECK-LABEL: add_v8i16_v8i32_sext: 80; CHECK: // %bb.0: // %entry 81; CHECK-NEXT: saddlv s0, v0.8h 82; CHECK-NEXT: fmov w0, s0 83; CHECK-NEXT: ret 84entry: 85 %xx = sext <8 x i16> %x to <8 x i32> 86 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) 87 ret i32 %z 88} 89 90define i32 @add_v4i16_v4i32_zext(<4 x i16> %x) { 91; CHECK-LABEL: add_v4i16_v4i32_zext: 92; CHECK: // %bb.0: // %entry 93; CHECK-NEXT: ushll v0.4s, v0.4h, #0 94; CHECK-NEXT: addv s0, v0.4s 95; CHECK-NEXT: fmov w0, s0 96; CHECK-NEXT: ret 97entry: 98 %xx = zext <4 x i16> %x to <4 x i32> 99 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) 100 ret i32 %z 101} 102 103define i32 @add_v4i16_v4i32_sext(<4 x i16> %x) { 104; CHECK-LABEL: add_v4i16_v4i32_sext: 105; CHECK: // %bb.0: // %entry 106; CHECK-NEXT: sshll v0.4s, v0.4h, #0 107; CHECK-NEXT: addv s0, v0.4s 108; CHECK-NEXT: fmov w0, s0 109; CHECK-NEXT: ret 110entry: 111 %xx = sext <4 x i16> %x to <4 x i32> 112 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) 113 ret i32 %z 114} 115 116define zeroext i16 @add_v8i16_v8i16(<8 x i16> %x) { 117; CHECK-LABEL: add_v8i16_v8i16: 118; CHECK: // %bb.0: // %entry 119; CHECK-NEXT: addv h0, v0.8h 120; CHECK-NEXT: fmov w0, s0 121; CHECK-NEXT: ret 122entry: 123 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %x) 124 ret i16 %z 125} 126 127define i64 @add_v8i16_v8i64_zext(<8 x i16> %x) { 128; CHECK-LABEL: add_v8i16_v8i64_zext: 129; CHECK: // %bb.0: // %entry 130; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0 131; CHECK-NEXT: ushll v0.4s, v0.4h, #0 132; CHECK-NEXT: uaddl2 v2.2d, v0.4s, v1.4s 133; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s 134; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 135; CHECK-NEXT: addp d0, v0.2d 136; CHECK-NEXT: fmov x0, d0 137; CHECK-NEXT: ret 138entry: 139 %xx = zext <8 x i16> %x to <8 x i64> 140 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) 141 ret i64 %z 142} 143 144define i64 @add_v8i16_v8i64_sext(<8 x i16> %x) { 145; CHECK-LABEL: add_v8i16_v8i64_sext: 146; CHECK: // %bb.0: // %entry 147; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0 148; CHECK-NEXT: sshll v0.4s, v0.4h, #0 149; CHECK-NEXT: saddl2 v2.2d, v0.4s, v1.4s 150; CHECK-NEXT: saddl v0.2d, v0.2s, v1.2s 151; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 152; CHECK-NEXT: addp d0, v0.2d 153; CHECK-NEXT: fmov x0, d0 154; CHECK-NEXT: ret 155entry: 156 %xx = sext <8 x i16> %x to <8 x i64> 157 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) 158 ret i64 %z 159} 160 161define i64 @add_v4i16_v4i64_zext(<4 x i16> %x) { 162; CHECK-LABEL: add_v4i16_v4i64_zext: 163; CHECK: // %bb.0: // %entry 164; CHECK-NEXT: ushll v0.4s, v0.4h, #0 165; CHECK-NEXT: uaddlv d0, v0.4s 166; CHECK-NEXT: fmov x0, d0 167; CHECK-NEXT: ret 168entry: 169 %xx = zext <4 x i16> %x to <4 x i64> 170 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 171 ret i64 %z 172} 173 174define i64 @add_v4i16_v4i64_sext(<4 x i16> %x) { 175; CHECK-LABEL: add_v4i16_v4i64_sext: 176; CHECK: // %bb.0: // %entry 177; CHECK-NEXT: sshll v0.4s, v0.4h, #0 178; CHECK-NEXT: saddlv d0, v0.4s 179; CHECK-NEXT: fmov x0, d0 180; CHECK-NEXT: ret 181entry: 182 %xx = sext <4 x i16> %x to <4 x i64> 183 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 184 ret i64 %z 185} 186 187define i64 @add_v2i16_v2i64_zext(<2 x i16> %x) { 188; CHECK-LABEL: add_v2i16_v2i64_zext: 189; CHECK: // %bb.0: // %entry 190; CHECK-NEXT: movi d1, #0x00ffff0000ffff 191; CHECK-NEXT: and v0.8b, v0.8b, v1.8b 192; CHECK-NEXT: ushll v0.2d, v0.2s, #0 193; CHECK-NEXT: addp d0, v0.2d 194; CHECK-NEXT: fmov x0, d0 195; CHECK-NEXT: ret 196entry: 197 %xx = zext <2 x i16> %x to <2 x i64> 198 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 199 ret i64 %z 200} 201 202define i64 @add_v2i16_v2i64_sext(<2 x i16> %x) { 203; CHECK-LABEL: add_v2i16_v2i64_sext: 204; CHECK: // %bb.0: // %entry 205; CHECK-NEXT: ushll v0.2d, v0.2s, #0 206; CHECK-NEXT: shl v0.2d, v0.2d, #48 207; CHECK-NEXT: sshr v0.2d, v0.2d, #48 208; CHECK-NEXT: addp d0, v0.2d 209; CHECK-NEXT: fmov x0, d0 210; CHECK-NEXT: ret 211entry: 212 %xx = sext <2 x i16> %x to <2 x i64> 213 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 214 ret i64 %z 215} 216 217define i32 @add_v16i8_v16i32_zext(<16 x i8> %x) { 218; CHECK-BASE-LABEL: add_v16i8_v16i32_zext: 219; CHECK-BASE: // %bb.0: // %entry 220; CHECK-BASE-NEXT: ushll2 v1.8h, v0.16b, #0 221; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0 222; CHECK-BASE-NEXT: uaddl2 v2.4s, v0.8h, v1.8h 223; CHECK-BASE-NEXT: uaddl v0.4s, v0.4h, v1.4h 224; CHECK-BASE-NEXT: add v0.4s, v0.4s, v2.4s 225; CHECK-BASE-NEXT: addv s0, v0.4s 226; CHECK-BASE-NEXT: fmov w0, s0 227; CHECK-BASE-NEXT: ret 228; 229; CHECK-DOT-LABEL: add_v16i8_v16i32_zext: 230; CHECK-DOT: // %bb.0: // %entry 231; CHECK-DOT-NEXT: movi v1.16b, #1 232; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 233; CHECK-DOT-NEXT: udot v2.4s, v0.16b, v1.16b 234; CHECK-DOT-NEXT: addv s0, v2.4s 235; CHECK-DOT-NEXT: fmov w0, s0 236; CHECK-DOT-NEXT: ret 237entry: 238 %xx = zext <16 x i8> %x to <16 x i32> 239 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx) 240 ret i32 %z 241} 242 243define i32 @add_v16i8_v16i32_sext(<16 x i8> %x) { 244; CHECK-BASE-LABEL: add_v16i8_v16i32_sext: 245; CHECK-BASE: // %bb.0: // %entry 246; CHECK-BASE-NEXT: sshll2 v1.8h, v0.16b, #0 247; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0 248; CHECK-BASE-NEXT: saddl2 v2.4s, v0.8h, v1.8h 249; CHECK-BASE-NEXT: saddl v0.4s, v0.4h, v1.4h 250; CHECK-BASE-NEXT: add v0.4s, v0.4s, v2.4s 251; CHECK-BASE-NEXT: addv s0, v0.4s 252; CHECK-BASE-NEXT: fmov w0, s0 253; CHECK-BASE-NEXT: ret 254; 255; CHECK-DOT-LABEL: add_v16i8_v16i32_sext: 256; CHECK-DOT: // %bb.0: // %entry 257; CHECK-DOT-NEXT: movi v1.16b, #1 258; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 259; CHECK-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b 260; CHECK-DOT-NEXT: addv s0, v2.4s 261; CHECK-DOT-NEXT: fmov w0, s0 262; CHECK-DOT-NEXT: ret 263entry: 264 %xx = sext <16 x i8> %x to <16 x i32> 265 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx) 266 ret i32 %z 267} 268 269define i32 @add_v8i8_v8i32_zext(<8 x i8> %x) { 270; CHECK-BASE-LABEL: add_v8i8_v8i32_zext: 271; CHECK-BASE: // %bb.0: // %entry 272; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0 273; CHECK-BASE-NEXT: uaddlv s0, v0.8h 274; CHECK-BASE-NEXT: fmov w0, s0 275; CHECK-BASE-NEXT: ret 276; 277; CHECK-DOT-LABEL: add_v8i8_v8i32_zext: 278; CHECK-DOT: // %bb.0: // %entry 279; CHECK-DOT-NEXT: movi v1.8b, #1 280; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 281; CHECK-DOT-NEXT: udot v2.2s, v0.8b, v1.8b 282; CHECK-DOT-NEXT: addp v0.2s, v2.2s, v2.2s 283; CHECK-DOT-NEXT: fmov w0, s0 284; CHECK-DOT-NEXT: ret 285entry: 286 %xx = zext <8 x i8> %x to <8 x i32> 287 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) 288 ret i32 %z 289} 290 291define i32 @add_v8i8_v8i32_sext(<8 x i8> %x) { 292; CHECK-BASE-LABEL: add_v8i8_v8i32_sext: 293; CHECK-BASE: // %bb.0: // %entry 294; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0 295; CHECK-BASE-NEXT: saddlv s0, v0.8h 296; CHECK-BASE-NEXT: fmov w0, s0 297; CHECK-BASE-NEXT: ret 298; 299; CHECK-DOT-LABEL: add_v8i8_v8i32_sext: 300; CHECK-DOT: // %bb.0: // %entry 301; CHECK-DOT-NEXT: movi v1.8b, #1 302; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 303; CHECK-DOT-NEXT: sdot v2.2s, v0.8b, v1.8b 304; CHECK-DOT-NEXT: addp v0.2s, v2.2s, v2.2s 305; CHECK-DOT-NEXT: fmov w0, s0 306; CHECK-DOT-NEXT: ret 307entry: 308 %xx = sext <8 x i8> %x to <8 x i32> 309 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) 310 ret i32 %z 311} 312 313define i32 @add_v4i8_v4i32_zext(<4 x i8> %x) { 314; CHECK-LABEL: add_v4i8_v4i32_zext: 315; CHECK: // %bb.0: // %entry 316; CHECK-NEXT: bic v0.4h, #255, lsl #8 317; CHECK-NEXT: ushll v0.4s, v0.4h, #0 318; CHECK-NEXT: addv s0, v0.4s 319; CHECK-NEXT: fmov w0, s0 320; CHECK-NEXT: ret 321entry: 322 %xx = zext <4 x i8> %x to <4 x i32> 323 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) 324 ret i32 %z 325} 326 327define i32 @add_v4i8_v4i32_sext(<4 x i8> %x) { 328; CHECK-LABEL: add_v4i8_v4i32_sext: 329; CHECK: // %bb.0: // %entry 330; CHECK-NEXT: ushll v0.4s, v0.4h, #0 331; CHECK-NEXT: shl v0.4s, v0.4s, #24 332; CHECK-NEXT: sshr v0.4s, v0.4s, #24 333; CHECK-NEXT: addv s0, v0.4s 334; CHECK-NEXT: fmov w0, s0 335; CHECK-NEXT: ret 336entry: 337 %xx = sext <4 x i8> %x to <4 x i32> 338 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) 339 ret i32 %z 340} 341 342define zeroext i16 @add_v16i8_v16i16_zext(<16 x i8> %x) { 343; CHECK-LABEL: add_v16i8_v16i16_zext: 344; CHECK: // %bb.0: // %entry 345; CHECK-NEXT: uaddlp v0.8h, v0.16b 346; CHECK-NEXT: addv h0, v0.8h 347; CHECK-NEXT: fmov w0, s0 348; CHECK-NEXT: ret 349entry: 350 %xx = zext <16 x i8> %x to <16 x i16> 351 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx) 352 ret i16 %z 353} 354 355define signext i16 @add_v16i8_v16i16_sext(<16 x i8> %x) { 356; CHECK-LABEL: add_v16i8_v16i16_sext: 357; CHECK: // %bb.0: // %entry 358; CHECK-NEXT: saddlp v0.8h, v0.16b 359; CHECK-NEXT: addv h0, v0.8h 360; CHECK-NEXT: smov w0, v0.h[0] 361; CHECK-NEXT: ret 362entry: 363 %xx = sext <16 x i8> %x to <16 x i16> 364 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx) 365 ret i16 %z 366} 367 368define zeroext i16 @add_v8i8_v8i16_zext(<8 x i8> %x) { 369; CHECK-LABEL: add_v8i8_v8i16_zext: 370; CHECK: // %bb.0: // %entry 371; CHECK-NEXT: ushll v0.8h, v0.8b, #0 372; CHECK-NEXT: addv h0, v0.8h 373; CHECK-NEXT: fmov w0, s0 374; CHECK-NEXT: ret 375entry: 376 %xx = zext <8 x i8> %x to <8 x i16> 377 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx) 378 ret i16 %z 379} 380 381define signext i16 @add_v8i8_v8i16_sext(<8 x i8> %x) { 382; CHECK-LABEL: add_v8i8_v8i16_sext: 383; CHECK: // %bb.0: // %entry 384; CHECK-NEXT: sshll v0.8h, v0.8b, #0 385; CHECK-NEXT: addv h0, v0.8h 386; CHECK-NEXT: smov w0, v0.h[0] 387; CHECK-NEXT: ret 388entry: 389 %xx = sext <8 x i8> %x to <8 x i16> 390 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx) 391 ret i16 %z 392} 393 394define zeroext i8 @add_v16i8_v16i8(<16 x i8> %x) { 395; CHECK-LABEL: add_v16i8_v16i8: 396; CHECK: // %bb.0: // %entry 397; CHECK-NEXT: addv b0, v0.16b 398; CHECK-NEXT: fmov w0, s0 399; CHECK-NEXT: ret 400entry: 401 %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %x) 402 ret i8 %z 403} 404 405define i64 @add_v16i8_v16i64_zext(<16 x i8> %x) { 406; CHECK-LABEL: add_v16i8_v16i64_zext: 407; CHECK: // %bb.0: // %entry 408; CHECK-NEXT: ushll2 v1.8h, v0.16b, #0 409; CHECK-NEXT: ushll v0.8h, v0.8b, #0 410; CHECK-NEXT: ushll v2.4s, v1.4h, #0 411; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0 412; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0 413; CHECK-NEXT: ushll v0.4s, v0.4h, #0 414; CHECK-NEXT: uaddl2 v4.2d, v3.4s, v1.4s 415; CHECK-NEXT: uaddl2 v5.2d, v0.4s, v2.4s 416; CHECK-NEXT: uaddl v1.2d, v3.2s, v1.2s 417; CHECK-NEXT: uaddl v0.2d, v0.2s, v2.2s 418; CHECK-NEXT: add v2.2d, v5.2d, v4.2d 419; CHECK-NEXT: add v0.2d, v0.2d, v1.2d 420; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 421; CHECK-NEXT: addp d0, v0.2d 422; CHECK-NEXT: fmov x0, d0 423; CHECK-NEXT: ret 424entry: 425 %xx = zext <16 x i8> %x to <16 x i64> 426 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx) 427 ret i64 %z 428} 429 430define i64 @add_v16i8_v16i64_sext(<16 x i8> %x) { 431; CHECK-LABEL: add_v16i8_v16i64_sext: 432; CHECK: // %bb.0: // %entry 433; CHECK-NEXT: sshll2 v1.8h, v0.16b, #0 434; CHECK-NEXT: sshll v0.8h, v0.8b, #0 435; CHECK-NEXT: sshll v2.4s, v1.4h, #0 436; CHECK-NEXT: sshll2 v1.4s, v1.8h, #0 437; CHECK-NEXT: sshll2 v3.4s, v0.8h, #0 438; CHECK-NEXT: sshll v0.4s, v0.4h, #0 439; CHECK-NEXT: saddl2 v4.2d, v3.4s, v1.4s 440; CHECK-NEXT: saddl2 v5.2d, v0.4s, v2.4s 441; CHECK-NEXT: saddl v1.2d, v3.2s, v1.2s 442; CHECK-NEXT: saddl v0.2d, v0.2s, v2.2s 443; CHECK-NEXT: add v2.2d, v5.2d, v4.2d 444; CHECK-NEXT: add v0.2d, v0.2d, v1.2d 445; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 446; CHECK-NEXT: addp d0, v0.2d 447; CHECK-NEXT: fmov x0, d0 448; CHECK-NEXT: ret 449entry: 450 %xx = sext <16 x i8> %x to <16 x i64> 451 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx) 452 ret i64 %z 453} 454 455define i64 @add_v8i8_v8i64_zext(<8 x i8> %x) { 456; CHECK-LABEL: add_v8i8_v8i64_zext: 457; CHECK: // %bb.0: // %entry 458; CHECK-NEXT: ushll v0.8h, v0.8b, #0 459; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0 460; CHECK-NEXT: ushll v0.4s, v0.4h, #0 461; CHECK-NEXT: uaddl2 v2.2d, v0.4s, v1.4s 462; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s 463; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 464; CHECK-NEXT: addp d0, v0.2d 465; CHECK-NEXT: fmov x0, d0 466; CHECK-NEXT: ret 467entry: 468 %xx = zext <8 x i8> %x to <8 x i64> 469 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) 470 ret i64 %z 471} 472 473define i64 @add_v8i8_v8i64_sext(<8 x i8> %x) { 474; CHECK-LABEL: add_v8i8_v8i64_sext: 475; CHECK: // %bb.0: // %entry 476; CHECK-NEXT: sshll v0.8h, v0.8b, #0 477; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0 478; CHECK-NEXT: sshll v0.4s, v0.4h, #0 479; CHECK-NEXT: saddl2 v2.2d, v0.4s, v1.4s 480; CHECK-NEXT: saddl v0.2d, v0.2s, v1.2s 481; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 482; CHECK-NEXT: addp d0, v0.2d 483; CHECK-NEXT: fmov x0, d0 484; CHECK-NEXT: ret 485entry: 486 %xx = sext <8 x i8> %x to <8 x i64> 487 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) 488 ret i64 %z 489} 490 491define i64 @add_v4i8_v4i64_zext(<4 x i8> %x) { 492; CHECK-LABEL: add_v4i8_v4i64_zext: 493; CHECK: // %bb.0: // %entry 494; CHECK-NEXT: bic v0.4h, #255, lsl #8 495; CHECK-NEXT: ushll v0.4s, v0.4h, #0 496; CHECK-NEXT: uaddlv d0, v0.4s 497; CHECK-NEXT: fmov x0, d0 498; CHECK-NEXT: ret 499entry: 500 %xx = zext <4 x i8> %x to <4 x i64> 501 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 502 ret i64 %z 503} 504 505define i64 @add_v4i8_v4i64_sext(<4 x i8> %x) { 506; CHECK-LABEL: add_v4i8_v4i64_sext: 507; CHECK: // %bb.0: // %entry 508; CHECK-NEXT: ushll v0.4s, v0.4h, #0 509; CHECK-NEXT: ushll v1.2d, v0.2s, #0 510; CHECK-NEXT: ushll2 v0.2d, v0.4s, #0 511; CHECK-NEXT: shl v1.2d, v1.2d, #56 512; CHECK-NEXT: shl v0.2d, v0.2d, #56 513; CHECK-NEXT: sshr v1.2d, v1.2d, #56 514; CHECK-NEXT: ssra v1.2d, v0.2d, #56 515; CHECK-NEXT: addp d0, v1.2d 516; CHECK-NEXT: fmov x0, d0 517; CHECK-NEXT: ret 518entry: 519 %xx = sext <4 x i8> %x to <4 x i64> 520 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 521 ret i64 %z 522} 523 524define i64 @add_v2i8_v2i64_zext(<2 x i8> %x) { 525; CHECK-LABEL: add_v2i8_v2i64_zext: 526; CHECK: // %bb.0: // %entry 527; CHECK-NEXT: movi d1, #0x0000ff000000ff 528; CHECK-NEXT: and v0.8b, v0.8b, v1.8b 529; CHECK-NEXT: ushll v0.2d, v0.2s, #0 530; CHECK-NEXT: addp d0, v0.2d 531; CHECK-NEXT: fmov x0, d0 532; CHECK-NEXT: ret 533entry: 534 %xx = zext <2 x i8> %x to <2 x i64> 535 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 536 ret i64 %z 537} 538 539define i64 @add_v2i8_v2i64_sext(<2 x i8> %x) { 540; CHECK-LABEL: add_v2i8_v2i64_sext: 541; CHECK: // %bb.0: // %entry 542; CHECK-NEXT: ushll v0.2d, v0.2s, #0 543; CHECK-NEXT: shl v0.2d, v0.2d, #56 544; CHECK-NEXT: sshr v0.2d, v0.2d, #56 545; CHECK-NEXT: addp d0, v0.2d 546; CHECK-NEXT: fmov x0, d0 547; CHECK-NEXT: ret 548entry: 549 %xx = sext <2 x i8> %x to <2 x i64> 550 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 551 ret i64 %z 552} 553 554define i64 @add_v2i64_v2i64(<2 x i64> %x) { 555; CHECK-LABEL: add_v2i64_v2i64: 556; CHECK: // %bb.0: // %entry 557; CHECK-NEXT: addp d0, v0.2d 558; CHECK-NEXT: fmov x0, d0 559; CHECK-NEXT: ret 560entry: 561 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %x) 562 ret i64 %z 563} 564 565define i32 @add_v4i32_v4i32_acc(<4 x i32> %x, i32 %a) { 566; CHECK-LABEL: add_v4i32_v4i32_acc: 567; CHECK: // %bb.0: // %entry 568; CHECK-NEXT: addv s0, v0.4s 569; CHECK-NEXT: fmov w8, s0 570; CHECK-NEXT: add w0, w8, w0 571; CHECK-NEXT: ret 572entry: 573 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x) 574 %r = add i32 %z, %a 575 ret i32 %r 576} 577 578define i64 @add_v4i32_v4i64_acc_zext(<4 x i32> %x, i64 %a) { 579; CHECK-LABEL: add_v4i32_v4i64_acc_zext: 580; CHECK: // %bb.0: // %entry 581; CHECK-NEXT: uaddlv d0, v0.4s 582; CHECK-NEXT: fmov x8, d0 583; CHECK-NEXT: add x0, x8, x0 584; CHECK-NEXT: ret 585entry: 586 %xx = zext <4 x i32> %x to <4 x i64> 587 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 588 %r = add i64 %z, %a 589 ret i64 %r 590} 591 592define i64 @add_v4i32_v4i64_acc_sext(<4 x i32> %x, i64 %a) { 593; CHECK-LABEL: add_v4i32_v4i64_acc_sext: 594; CHECK: // %bb.0: // %entry 595; CHECK-NEXT: saddlv d0, v0.4s 596; CHECK-NEXT: fmov x8, d0 597; CHECK-NEXT: add x0, x8, x0 598; CHECK-NEXT: ret 599entry: 600 %xx = sext <4 x i32> %x to <4 x i64> 601 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 602 %r = add i64 %z, %a 603 ret i64 %r 604} 605 606define i64 @add_v2i32_v2i64_acc_zext(<2 x i32> %x, i64 %a) { 607; CHECK-LABEL: add_v2i32_v2i64_acc_zext: 608; CHECK: // %bb.0: // %entry 609; CHECK-NEXT: ushll v0.2d, v0.2s, #0 610; CHECK-NEXT: addp d0, v0.2d 611; CHECK-NEXT: fmov x8, d0 612; CHECK-NEXT: add x0, x8, x0 613; CHECK-NEXT: ret 614entry: 615 %xx = zext <2 x i32> %x to <2 x i64> 616 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 617 %r = add i64 %z, %a 618 ret i64 %r 619} 620 621define i64 @add_v2i32_v2i64_acc_sext(<2 x i32> %x, i64 %a) { 622; CHECK-LABEL: add_v2i32_v2i64_acc_sext: 623; CHECK: // %bb.0: // %entry 624; CHECK-NEXT: sshll v0.2d, v0.2s, #0 625; CHECK-NEXT: addp d0, v0.2d 626; CHECK-NEXT: fmov x8, d0 627; CHECK-NEXT: add x0, x8, x0 628; CHECK-NEXT: ret 629entry: 630 %xx = sext <2 x i32> %x to <2 x i64> 631 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 632 %r = add i64 %z, %a 633 ret i64 %r 634} 635 636define i32 @add_v8i16_v8i32_acc_zext(<8 x i16> %x, i32 %a) { 637; CHECK-LABEL: add_v8i16_v8i32_acc_zext: 638; CHECK: // %bb.0: // %entry 639; CHECK-NEXT: uaddlv s0, v0.8h 640; CHECK-NEXT: fmov w8, s0 641; CHECK-NEXT: add w0, w8, w0 642; CHECK-NEXT: ret 643entry: 644 %xx = zext <8 x i16> %x to <8 x i32> 645 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) 646 %r = add i32 %z, %a 647 ret i32 %r 648} 649 650define i32 @add_v8i16_v8i32_acc_sext(<8 x i16> %x, i32 %a) { 651; CHECK-LABEL: add_v8i16_v8i32_acc_sext: 652; CHECK: // %bb.0: // %entry 653; CHECK-NEXT: saddlv s0, v0.8h 654; CHECK-NEXT: fmov w8, s0 655; CHECK-NEXT: add w0, w8, w0 656; CHECK-NEXT: ret 657entry: 658 %xx = sext <8 x i16> %x to <8 x i32> 659 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) 660 %r = add i32 %z, %a 661 ret i32 %r 662} 663 664define i32 @add_v4i16_v4i32_acc_zext(<4 x i16> %x, i32 %a) { 665; CHECK-LABEL: add_v4i16_v4i32_acc_zext: 666; CHECK: // %bb.0: // %entry 667; CHECK-NEXT: ushll v0.4s, v0.4h, #0 668; CHECK-NEXT: addv s0, v0.4s 669; CHECK-NEXT: fmov w8, s0 670; CHECK-NEXT: add w0, w8, w0 671; CHECK-NEXT: ret 672entry: 673 %xx = zext <4 x i16> %x to <4 x i32> 674 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) 675 %r = add i32 %z, %a 676 ret i32 %r 677} 678 679define i32 @add_v4i16_v4i32_acc_sext(<4 x i16> %x, i32 %a) { 680; CHECK-LABEL: add_v4i16_v4i32_acc_sext: 681; CHECK: // %bb.0: // %entry 682; CHECK-NEXT: sshll v0.4s, v0.4h, #0 683; CHECK-NEXT: addv s0, v0.4s 684; CHECK-NEXT: fmov w8, s0 685; CHECK-NEXT: add w0, w8, w0 686; CHECK-NEXT: ret 687entry: 688 %xx = sext <4 x i16> %x to <4 x i32> 689 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) 690 %r = add i32 %z, %a 691 ret i32 %r 692} 693 694define zeroext i16 @add_v8i16_v8i16_acc(<8 x i16> %x, i16 %a) { 695; CHECK-LABEL: add_v8i16_v8i16_acc: 696; CHECK: // %bb.0: // %entry 697; CHECK-NEXT: addv h0, v0.8h 698; CHECK-NEXT: fmov w8, s0 699; CHECK-NEXT: add w8, w8, w0 700; CHECK-NEXT: and w0, w8, #0xffff 701; CHECK-NEXT: ret 702entry: 703 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %x) 704 %r = add i16 %z, %a 705 ret i16 %r 706} 707 708define i64 @add_v8i16_v8i64_acc_zext(<8 x i16> %x, i64 %a) { 709; CHECK-LABEL: add_v8i16_v8i64_acc_zext: 710; CHECK: // %bb.0: // %entry 711; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0 712; CHECK-NEXT: ushll v0.4s, v0.4h, #0 713; CHECK-NEXT: uaddl2 v2.2d, v0.4s, v1.4s 714; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s 715; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 716; CHECK-NEXT: addp d0, v0.2d 717; CHECK-NEXT: fmov x8, d0 718; CHECK-NEXT: add x0, x8, x0 719; CHECK-NEXT: ret 720entry: 721 %xx = zext <8 x i16> %x to <8 x i64> 722 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) 723 %r = add i64 %z, %a 724 ret i64 %r 725} 726 727define i64 @add_v8i16_v8i64_acc_sext(<8 x i16> %x, i64 %a) { 728; CHECK-LABEL: add_v8i16_v8i64_acc_sext: 729; CHECK: // %bb.0: // %entry 730; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0 731; CHECK-NEXT: sshll v0.4s, v0.4h, #0 732; CHECK-NEXT: saddl2 v2.2d, v0.4s, v1.4s 733; CHECK-NEXT: saddl v0.2d, v0.2s, v1.2s 734; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 735; CHECK-NEXT: addp d0, v0.2d 736; CHECK-NEXT: fmov x8, d0 737; CHECK-NEXT: add x0, x8, x0 738; CHECK-NEXT: ret 739entry: 740 %xx = sext <8 x i16> %x to <8 x i64> 741 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) 742 %r = add i64 %z, %a 743 ret i64 %r 744} 745 746define i64 @add_v4i16_v4i64_acc_zext(<4 x i16> %x, i64 %a) { 747; CHECK-LABEL: add_v4i16_v4i64_acc_zext: 748; CHECK: // %bb.0: // %entry 749; CHECK-NEXT: ushll v0.4s, v0.4h, #0 750; CHECK-NEXT: uaddlv d0, v0.4s 751; CHECK-NEXT: fmov x8, d0 752; CHECK-NEXT: add x0, x8, x0 753; CHECK-NEXT: ret 754entry: 755 %xx = zext <4 x i16> %x to <4 x i64> 756 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 757 %r = add i64 %z, %a 758 ret i64 %r 759} 760 761define i64 @add_v4i16_v4i64_acc_sext(<4 x i16> %x, i64 %a) { 762; CHECK-LABEL: add_v4i16_v4i64_acc_sext: 763; CHECK: // %bb.0: // %entry 764; CHECK-NEXT: sshll v0.4s, v0.4h, #0 765; CHECK-NEXT: saddlv d0, v0.4s 766; CHECK-NEXT: fmov x8, d0 767; CHECK-NEXT: add x0, x8, x0 768; CHECK-NEXT: ret 769entry: 770 %xx = sext <4 x i16> %x to <4 x i64> 771 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 772 %r = add i64 %z, %a 773 ret i64 %r 774} 775 776define i64 @add_v2i16_v2i64_acc_zext(<2 x i16> %x, i64 %a) { 777; CHECK-LABEL: add_v2i16_v2i64_acc_zext: 778; CHECK: // %bb.0: // %entry 779; CHECK-NEXT: movi d1, #0x00ffff0000ffff 780; CHECK-NEXT: and v0.8b, v0.8b, v1.8b 781; CHECK-NEXT: ushll v0.2d, v0.2s, #0 782; CHECK-NEXT: addp d0, v0.2d 783; CHECK-NEXT: fmov x8, d0 784; CHECK-NEXT: add x0, x8, x0 785; CHECK-NEXT: ret 786entry: 787 %xx = zext <2 x i16> %x to <2 x i64> 788 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 789 %r = add i64 %z, %a 790 ret i64 %r 791} 792 793define i64 @add_v2i16_v2i64_acc_sext(<2 x i16> %x, i64 %a) { 794; CHECK-LABEL: add_v2i16_v2i64_acc_sext: 795; CHECK: // %bb.0: // %entry 796; CHECK-NEXT: ushll v0.2d, v0.2s, #0 797; CHECK-NEXT: shl v0.2d, v0.2d, #48 798; CHECK-NEXT: sshr v0.2d, v0.2d, #48 799; CHECK-NEXT: addp d0, v0.2d 800; CHECK-NEXT: fmov x8, d0 801; CHECK-NEXT: add x0, x8, x0 802; CHECK-NEXT: ret 803entry: 804 %xx = sext <2 x i16> %x to <2 x i64> 805 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 806 %r = add i64 %z, %a 807 ret i64 %r 808} 809 810define i32 @add_v16i8_v16i32_acc_zext(<16 x i8> %x, i32 %a) { 811; CHECK-BASE-LABEL: add_v16i8_v16i32_acc_zext: 812; CHECK-BASE: // %bb.0: // %entry 813; CHECK-BASE-NEXT: ushll2 v1.8h, v0.16b, #0 814; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0 815; CHECK-BASE-NEXT: uaddl2 v2.4s, v0.8h, v1.8h 816; CHECK-BASE-NEXT: uaddl v0.4s, v0.4h, v1.4h 817; CHECK-BASE-NEXT: add v0.4s, v0.4s, v2.4s 818; CHECK-BASE-NEXT: addv s0, v0.4s 819; CHECK-BASE-NEXT: fmov w8, s0 820; CHECK-BASE-NEXT: add w0, w8, w0 821; CHECK-BASE-NEXT: ret 822; 823; CHECK-DOT-LABEL: add_v16i8_v16i32_acc_zext: 824; CHECK-DOT: // %bb.0: // %entry 825; CHECK-DOT-NEXT: movi v1.16b, #1 826; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 827; CHECK-DOT-NEXT: udot v2.4s, v0.16b, v1.16b 828; CHECK-DOT-NEXT: addv s0, v2.4s 829; CHECK-DOT-NEXT: fmov w8, s0 830; CHECK-DOT-NEXT: add w0, w8, w0 831; CHECK-DOT-NEXT: ret 832entry: 833 %xx = zext <16 x i8> %x to <16 x i32> 834 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx) 835 %r = add i32 %z, %a 836 ret i32 %r 837} 838 839define i32 @add_v16i8_v16i32_acc_sext(<16 x i8> %x, i32 %a) { 840; CHECK-BASE-LABEL: add_v16i8_v16i32_acc_sext: 841; CHECK-BASE: // %bb.0: // %entry 842; CHECK-BASE-NEXT: sshll2 v1.8h, v0.16b, #0 843; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0 844; CHECK-BASE-NEXT: saddl2 v2.4s, v0.8h, v1.8h 845; CHECK-BASE-NEXT: saddl v0.4s, v0.4h, v1.4h 846; CHECK-BASE-NEXT: add v0.4s, v0.4s, v2.4s 847; CHECK-BASE-NEXT: addv s0, v0.4s 848; CHECK-BASE-NEXT: fmov w8, s0 849; CHECK-BASE-NEXT: add w0, w8, w0 850; CHECK-BASE-NEXT: ret 851; 852; CHECK-DOT-LABEL: add_v16i8_v16i32_acc_sext: 853; CHECK-DOT: // %bb.0: // %entry 854; CHECK-DOT-NEXT: movi v1.16b, #1 855; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 856; CHECK-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b 857; CHECK-DOT-NEXT: addv s0, v2.4s 858; CHECK-DOT-NEXT: fmov w8, s0 859; CHECK-DOT-NEXT: add w0, w8, w0 860; CHECK-DOT-NEXT: ret 861entry: 862 %xx = sext <16 x i8> %x to <16 x i32> 863 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx) 864 %r = add i32 %z, %a 865 ret i32 %r 866} 867 868define i32 @add_v8i8_v8i32_acc_zext(<8 x i8> %x, i32 %a) { 869; CHECK-BASE-LABEL: add_v8i8_v8i32_acc_zext: 870; CHECK-BASE: // %bb.0: // %entry 871; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0 872; CHECK-BASE-NEXT: uaddlv s0, v0.8h 873; CHECK-BASE-NEXT: fmov w8, s0 874; CHECK-BASE-NEXT: add w0, w8, w0 875; CHECK-BASE-NEXT: ret 876; 877; CHECK-DOT-LABEL: add_v8i8_v8i32_acc_zext: 878; CHECK-DOT: // %bb.0: // %entry 879; CHECK-DOT-NEXT: movi v1.8b, #1 880; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 881; CHECK-DOT-NEXT: udot v2.2s, v0.8b, v1.8b 882; CHECK-DOT-NEXT: addp v0.2s, v2.2s, v2.2s 883; CHECK-DOT-NEXT: fmov w8, s0 884; CHECK-DOT-NEXT: add w0, w8, w0 885; CHECK-DOT-NEXT: ret 886entry: 887 %xx = zext <8 x i8> %x to <8 x i32> 888 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) 889 %r = add i32 %z, %a 890 ret i32 %r 891} 892 893define i32 @add_v8i8_v8i32_acc_sext(<8 x i8> %x, i32 %a) { 894; CHECK-BASE-LABEL: add_v8i8_v8i32_acc_sext: 895; CHECK-BASE: // %bb.0: // %entry 896; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0 897; CHECK-BASE-NEXT: saddlv s0, v0.8h 898; CHECK-BASE-NEXT: fmov w8, s0 899; CHECK-BASE-NEXT: add w0, w8, w0 900; CHECK-BASE-NEXT: ret 901; 902; CHECK-DOT-LABEL: add_v8i8_v8i32_acc_sext: 903; CHECK-DOT: // %bb.0: // %entry 904; CHECK-DOT-NEXT: movi v1.8b, #1 905; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 906; CHECK-DOT-NEXT: sdot v2.2s, v0.8b, v1.8b 907; CHECK-DOT-NEXT: addp v0.2s, v2.2s, v2.2s 908; CHECK-DOT-NEXT: fmov w8, s0 909; CHECK-DOT-NEXT: add w0, w8, w0 910; CHECK-DOT-NEXT: ret 911entry: 912 %xx = sext <8 x i8> %x to <8 x i32> 913 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) 914 %r = add i32 %z, %a 915 ret i32 %r 916} 917 918define i32 @add_v4i8_v4i32_acc_zext(<4 x i8> %x, i32 %a) { 919; CHECK-LABEL: add_v4i8_v4i32_acc_zext: 920; CHECK: // %bb.0: // %entry 921; CHECK-NEXT: bic v0.4h, #255, lsl #8 922; CHECK-NEXT: ushll v0.4s, v0.4h, #0 923; CHECK-NEXT: addv s0, v0.4s 924; CHECK-NEXT: fmov w8, s0 925; CHECK-NEXT: add w0, w8, w0 926; CHECK-NEXT: ret 927entry: 928 %xx = zext <4 x i8> %x to <4 x i32> 929 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) 930 %r = add i32 %z, %a 931 ret i32 %r 932} 933 934define i32 @add_v4i8_v4i32_acc_sext(<4 x i8> %x, i32 %a) { 935; CHECK-LABEL: add_v4i8_v4i32_acc_sext: 936; CHECK: // %bb.0: // %entry 937; CHECK-NEXT: ushll v0.4s, v0.4h, #0 938; CHECK-NEXT: shl v0.4s, v0.4s, #24 939; CHECK-NEXT: sshr v0.4s, v0.4s, #24 940; CHECK-NEXT: addv s0, v0.4s 941; CHECK-NEXT: fmov w8, s0 942; CHECK-NEXT: add w0, w8, w0 943; CHECK-NEXT: ret 944entry: 945 %xx = sext <4 x i8> %x to <4 x i32> 946 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) 947 %r = add i32 %z, %a 948 ret i32 %r 949} 950 951define zeroext i16 @add_v16i8_v16i16_acc_zext(<16 x i8> %x, i16 %a) { 952; CHECK-LABEL: add_v16i8_v16i16_acc_zext: 953; CHECK: // %bb.0: // %entry 954; CHECK-NEXT: uaddlv h0, v0.16b 955; CHECK-NEXT: fmov w8, s0 956; CHECK-NEXT: add w8, w8, w0 957; CHECK-NEXT: and w0, w8, #0xffff 958; CHECK-NEXT: ret 959entry: 960 %xx = zext <16 x i8> %x to <16 x i16> 961 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx) 962 %r = add i16 %z, %a 963 ret i16 %r 964} 965 966define signext i16 @add_v16i8_v16i16_acc_sext(<16 x i8> %x, i16 %a) { 967; CHECK-LABEL: add_v16i8_v16i16_acc_sext: 968; CHECK: // %bb.0: // %entry 969; CHECK-NEXT: saddlv h0, v0.16b 970; CHECK-NEXT: fmov w8, s0 971; CHECK-NEXT: add w8, w8, w0 972; CHECK-NEXT: sxth w0, w8 973; CHECK-NEXT: ret 974entry: 975 %xx = sext <16 x i8> %x to <16 x i16> 976 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx) 977 %r = add i16 %z, %a 978 ret i16 %r 979} 980 981define zeroext i16 @add_v8i8_v8i16_acc_zext(<8 x i8> %x, i16 %a) { 982; CHECK-LABEL: add_v8i8_v8i16_acc_zext: 983; CHECK: // %bb.0: // %entry 984; CHECK-NEXT: ushll v0.8h, v0.8b, #0 985; CHECK-NEXT: addv h0, v0.8h 986; CHECK-NEXT: fmov w8, s0 987; CHECK-NEXT: add w8, w8, w0 988; CHECK-NEXT: and w0, w8, #0xffff 989; CHECK-NEXT: ret 990entry: 991 %xx = zext <8 x i8> %x to <8 x i16> 992 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx) 993 %r = add i16 %z, %a 994 ret i16 %r 995} 996 997define signext i16 @add_v8i8_v8i16_acc_sext(<8 x i8> %x, i16 %a) { 998; CHECK-LABEL: add_v8i8_v8i16_acc_sext: 999; CHECK: // %bb.0: // %entry 1000; CHECK-NEXT: sshll v0.8h, v0.8b, #0 1001; CHECK-NEXT: addv h0, v0.8h 1002; CHECK-NEXT: fmov w8, s0 1003; CHECK-NEXT: add w8, w8, w0 1004; CHECK-NEXT: sxth w0, w8 1005; CHECK-NEXT: ret 1006entry: 1007 %xx = sext <8 x i8> %x to <8 x i16> 1008 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx) 1009 %r = add i16 %z, %a 1010 ret i16 %r 1011} 1012 1013define zeroext i8 @add_v16i8_v16i8_acc(<16 x i8> %x, i8 %a) { 1014; CHECK-LABEL: add_v16i8_v16i8_acc: 1015; CHECK: // %bb.0: // %entry 1016; CHECK-NEXT: addv b0, v0.16b 1017; CHECK-NEXT: fmov w8, s0 1018; CHECK-NEXT: add w8, w8, w0 1019; CHECK-NEXT: and w0, w8, #0xff 1020; CHECK-NEXT: ret 1021entry: 1022 %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %x) 1023 %r = add i8 %z, %a 1024 ret i8 %r 1025} 1026 1027define i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, i64 %a) { 1028; CHECK-LABEL: add_v16i8_v16i64_acc_zext: 1029; CHECK: // %bb.0: // %entry 1030; CHECK-NEXT: ushll2 v1.8h, v0.16b, #0 1031; CHECK-NEXT: ushll v0.8h, v0.8b, #0 1032; CHECK-NEXT: ushll v2.4s, v1.4h, #0 1033; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0 1034; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0 1035; CHECK-NEXT: ushll v0.4s, v0.4h, #0 1036; CHECK-NEXT: uaddl2 v4.2d, v3.4s, v1.4s 1037; CHECK-NEXT: uaddl2 v5.2d, v0.4s, v2.4s 1038; CHECK-NEXT: uaddl v1.2d, v3.2s, v1.2s 1039; CHECK-NEXT: uaddl v0.2d, v0.2s, v2.2s 1040; CHECK-NEXT: add v2.2d, v5.2d, v4.2d 1041; CHECK-NEXT: add v0.2d, v0.2d, v1.2d 1042; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 1043; CHECK-NEXT: addp d0, v0.2d 1044; CHECK-NEXT: fmov x8, d0 1045; CHECK-NEXT: add x0, x8, x0 1046; CHECK-NEXT: ret 1047entry: 1048 %xx = zext <16 x i8> %x to <16 x i64> 1049 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx) 1050 %r = add i64 %z, %a 1051 ret i64 %r 1052} 1053 1054define i64 @add_v16i8_v16i64_acc_sext(<16 x i8> %x, i64 %a) { 1055; CHECK-LABEL: add_v16i8_v16i64_acc_sext: 1056; CHECK: // %bb.0: // %entry 1057; CHECK-NEXT: sshll2 v1.8h, v0.16b, #0 1058; CHECK-NEXT: sshll v0.8h, v0.8b, #0 1059; CHECK-NEXT: sshll v2.4s, v1.4h, #0 1060; CHECK-NEXT: sshll2 v1.4s, v1.8h, #0 1061; CHECK-NEXT: sshll2 v3.4s, v0.8h, #0 1062; CHECK-NEXT: sshll v0.4s, v0.4h, #0 1063; CHECK-NEXT: saddl2 v4.2d, v3.4s, v1.4s 1064; CHECK-NEXT: saddl2 v5.2d, v0.4s, v2.4s 1065; CHECK-NEXT: saddl v1.2d, v3.2s, v1.2s 1066; CHECK-NEXT: saddl v0.2d, v0.2s, v2.2s 1067; CHECK-NEXT: add v2.2d, v5.2d, v4.2d 1068; CHECK-NEXT: add v0.2d, v0.2d, v1.2d 1069; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 1070; CHECK-NEXT: addp d0, v0.2d 1071; CHECK-NEXT: fmov x8, d0 1072; CHECK-NEXT: add x0, x8, x0 1073; CHECK-NEXT: ret 1074entry: 1075 %xx = sext <16 x i8> %x to <16 x i64> 1076 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx) 1077 %r = add i64 %z, %a 1078 ret i64 %r 1079} 1080 1081define i64 @add_v8i8_v8i64_acc_zext(<8 x i8> %x, i64 %a) { 1082; CHECK-LABEL: add_v8i8_v8i64_acc_zext: 1083; CHECK: // %bb.0: // %entry 1084; CHECK-NEXT: ushll v0.8h, v0.8b, #0 1085; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0 1086; CHECK-NEXT: ushll v0.4s, v0.4h, #0 1087; CHECK-NEXT: uaddl2 v2.2d, v0.4s, v1.4s 1088; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s 1089; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 1090; CHECK-NEXT: addp d0, v0.2d 1091; CHECK-NEXT: fmov x8, d0 1092; CHECK-NEXT: add x0, x8, x0 1093; CHECK-NEXT: ret 1094entry: 1095 %xx = zext <8 x i8> %x to <8 x i64> 1096 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) 1097 %r = add i64 %z, %a 1098 ret i64 %r 1099} 1100 1101define i64 @add_v8i8_v8i64_acc_sext(<8 x i8> %x, i64 %a) { 1102; CHECK-LABEL: add_v8i8_v8i64_acc_sext: 1103; CHECK: // %bb.0: // %entry 1104; CHECK-NEXT: sshll v0.8h, v0.8b, #0 1105; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0 1106; CHECK-NEXT: sshll v0.4s, v0.4h, #0 1107; CHECK-NEXT: saddl2 v2.2d, v0.4s, v1.4s 1108; CHECK-NEXT: saddl v0.2d, v0.2s, v1.2s 1109; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 1110; CHECK-NEXT: addp d0, v0.2d 1111; CHECK-NEXT: fmov x8, d0 1112; CHECK-NEXT: add x0, x8, x0 1113; CHECK-NEXT: ret 1114entry: 1115 %xx = sext <8 x i8> %x to <8 x i64> 1116 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) 1117 %r = add i64 %z, %a 1118 ret i64 %r 1119} 1120 1121define i64 @add_v4i8_v4i64_acc_zext(<4 x i8> %x, i64 %a) { 1122; CHECK-LABEL: add_v4i8_v4i64_acc_zext: 1123; CHECK: // %bb.0: // %entry 1124; CHECK-NEXT: bic v0.4h, #255, lsl #8 1125; CHECK-NEXT: ushll v0.4s, v0.4h, #0 1126; CHECK-NEXT: uaddlv d0, v0.4s 1127; CHECK-NEXT: fmov x8, d0 1128; CHECK-NEXT: add x0, x8, x0 1129; CHECK-NEXT: ret 1130entry: 1131 %xx = zext <4 x i8> %x to <4 x i64> 1132 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 1133 %r = add i64 %z, %a 1134 ret i64 %r 1135} 1136 1137define i64 @add_v4i8_v4i64_acc_sext(<4 x i8> %x, i64 %a) { 1138; CHECK-LABEL: add_v4i8_v4i64_acc_sext: 1139; CHECK: // %bb.0: // %entry 1140; CHECK-NEXT: ushll v0.4s, v0.4h, #0 1141; CHECK-NEXT: ushll v1.2d, v0.2s, #0 1142; CHECK-NEXT: ushll2 v0.2d, v0.4s, #0 1143; CHECK-NEXT: shl v1.2d, v1.2d, #56 1144; CHECK-NEXT: shl v0.2d, v0.2d, #56 1145; CHECK-NEXT: sshr v1.2d, v1.2d, #56 1146; CHECK-NEXT: ssra v1.2d, v0.2d, #56 1147; CHECK-NEXT: addp d0, v1.2d 1148; CHECK-NEXT: fmov x8, d0 1149; CHECK-NEXT: add x0, x8, x0 1150; CHECK-NEXT: ret 1151entry: 1152 %xx = sext <4 x i8> %x to <4 x i64> 1153 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 1154 %r = add i64 %z, %a 1155 ret i64 %r 1156} 1157 1158define i64 @add_v2i8_v2i64_acc_zext(<2 x i8> %x, i64 %a) { 1159; CHECK-LABEL: add_v2i8_v2i64_acc_zext: 1160; CHECK: // %bb.0: // %entry 1161; CHECK-NEXT: movi d1, #0x0000ff000000ff 1162; CHECK-NEXT: and v0.8b, v0.8b, v1.8b 1163; CHECK-NEXT: ushll v0.2d, v0.2s, #0 1164; CHECK-NEXT: addp d0, v0.2d 1165; CHECK-NEXT: fmov x8, d0 1166; CHECK-NEXT: add x0, x8, x0 1167; CHECK-NEXT: ret 1168entry: 1169 %xx = zext <2 x i8> %x to <2 x i64> 1170 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 1171 %r = add i64 %z, %a 1172 ret i64 %r 1173} 1174 1175define i64 @add_v2i8_v2i64_acc_sext(<2 x i8> %x, i64 %a) { 1176; CHECK-LABEL: add_v2i8_v2i64_acc_sext: 1177; CHECK: // %bb.0: // %entry 1178; CHECK-NEXT: ushll v0.2d, v0.2s, #0 1179; CHECK-NEXT: shl v0.2d, v0.2d, #56 1180; CHECK-NEXT: sshr v0.2d, v0.2d, #56 1181; CHECK-NEXT: addp d0, v0.2d 1182; CHECK-NEXT: fmov x8, d0 1183; CHECK-NEXT: add x0, x8, x0 1184; CHECK-NEXT: ret 1185entry: 1186 %xx = sext <2 x i8> %x to <2 x i64> 1187 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 1188 %r = add i64 %z, %a 1189 ret i64 %r 1190} 1191 1192define i64 @add_v2i64_v2i64_acc(<2 x i64> %x, i64 %a) { 1193; CHECK-LABEL: add_v2i64_v2i64_acc: 1194; CHECK: // %bb.0: // %entry 1195; CHECK-NEXT: addp d0, v0.2d 1196; CHECK-NEXT: fmov x8, d0 1197; CHECK-NEXT: add x0, x8, x0 1198; CHECK-NEXT: ret 1199entry: 1200 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %x) 1201 %r = add i64 %z, %a 1202 ret i64 %r 1203} 1204 1205define i32 @add_pair_v4i32_v4i32(<4 x i32> %x, <4 x i32> %y) { 1206; CHECK-LABEL: add_pair_v4i32_v4i32: 1207; CHECK: // %bb.0: // %entry 1208; CHECK-NEXT: add v0.4s, v0.4s, v1.4s 1209; CHECK-NEXT: addv s0, v0.4s 1210; CHECK-NEXT: fmov w0, s0 1211; CHECK-NEXT: ret 1212entry: 1213 %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x) 1214 %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %y) 1215 %z = add i32 %z1, %z2 1216 ret i32 %z 1217} 1218 1219define i64 @add_pair_v4i32_v4i64_zext(<4 x i32> %x, <4 x i32> %y) { 1220; CHECK-LABEL: add_pair_v4i32_v4i64_zext: 1221; CHECK: // %bb.0: // %entry 1222; CHECK-NEXT: uaddlp v0.2d, v0.4s 1223; CHECK-NEXT: uadalp v0.2d, v1.4s 1224; CHECK-NEXT: addp d0, v0.2d 1225; CHECK-NEXT: fmov x0, d0 1226; CHECK-NEXT: ret 1227entry: 1228 %xx = zext <4 x i32> %x to <4 x i64> 1229 %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 1230 %yy = zext <4 x i32> %y to <4 x i64> 1231 %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy) 1232 %z = add i64 %z1, %z2 1233 ret i64 %z 1234} 1235 1236define i64 @add_pair_v4i32_v4i64_sext(<4 x i32> %x, <4 x i32> %y) { 1237; CHECK-LABEL: add_pair_v4i32_v4i64_sext: 1238; CHECK: // %bb.0: // %entry 1239; CHECK-NEXT: saddlp v0.2d, v0.4s 1240; CHECK-NEXT: sadalp v0.2d, v1.4s 1241; CHECK-NEXT: addp d0, v0.2d 1242; CHECK-NEXT: fmov x0, d0 1243; CHECK-NEXT: ret 1244entry: 1245 %xx = sext <4 x i32> %x to <4 x i64> 1246 %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 1247 %yy = sext <4 x i32> %y to <4 x i64> 1248 %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy) 1249 %z = add i64 %z1, %z2 1250 ret i64 %z 1251} 1252 1253define i64 @add_pair_v2i32_v2i64_zext(<2 x i32> %x, <2 x i32> %y) { 1254; CHECK-LABEL: add_pair_v2i32_v2i64_zext: 1255; CHECK: // %bb.0: // %entry 1256; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s 1257; CHECK-NEXT: addp d0, v0.2d 1258; CHECK-NEXT: fmov x0, d0 1259; CHECK-NEXT: ret 1260entry: 1261 %xx = zext <2 x i32> %x to <2 x i64> 1262 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 1263 %yy = zext <2 x i32> %y to <2 x i64> 1264 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy) 1265 %z = add i64 %z1, %z2 1266 ret i64 %z 1267} 1268 1269define i64 @add_pair_v2i32_v2i64_sext(<2 x i32> %x, <2 x i32> %y) { 1270; CHECK-LABEL: add_pair_v2i32_v2i64_sext: 1271; CHECK: // %bb.0: // %entry 1272; CHECK-NEXT: saddl v0.2d, v0.2s, v1.2s 1273; CHECK-NEXT: addp d0, v0.2d 1274; CHECK-NEXT: fmov x0, d0 1275; CHECK-NEXT: ret 1276entry: 1277 %xx = sext <2 x i32> %x to <2 x i64> 1278 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 1279 %yy = sext <2 x i32> %y to <2 x i64> 1280 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy) 1281 %z = add i64 %z1, %z2 1282 ret i64 %z 1283} 1284 1285define i32 @add_pair_v8i16_v8i32_zext(<8 x i16> %x, <8 x i16> %y) { 1286; CHECK-LABEL: add_pair_v8i16_v8i32_zext: 1287; CHECK: // %bb.0: // %entry 1288; CHECK-NEXT: uaddlp v0.4s, v0.8h 1289; CHECK-NEXT: uadalp v0.4s, v1.8h 1290; CHECK-NEXT: addv s0, v0.4s 1291; CHECK-NEXT: fmov w0, s0 1292; CHECK-NEXT: ret 1293entry: 1294 %xx = zext <8 x i16> %x to <8 x i32> 1295 %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) 1296 %yy = zext <8 x i16> %y to <8 x i32> 1297 %z2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %yy) 1298 %z = add i32 %z1, %z2 1299 ret i32 %z 1300} 1301 1302define i32 @add_pair_v8i16_v8i32_sext(<8 x i16> %x, <8 x i16> %y) { 1303; CHECK-LABEL: add_pair_v8i16_v8i32_sext: 1304; CHECK: // %bb.0: // %entry 1305; CHECK-NEXT: saddlp v0.4s, v0.8h 1306; CHECK-NEXT: sadalp v0.4s, v1.8h 1307; CHECK-NEXT: addv s0, v0.4s 1308; CHECK-NEXT: fmov w0, s0 1309; CHECK-NEXT: ret 1310entry: 1311 %xx = sext <8 x i16> %x to <8 x i32> 1312 %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) 1313 %yy = sext <8 x i16> %y to <8 x i32> 1314 %z2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %yy) 1315 %z = add i32 %z1, %z2 1316 ret i32 %z 1317} 1318 1319define i32 @add_pair_v4i16_v4i32_zext(<4 x i16> %x, <4 x i16> %y) { 1320; CHECK-LABEL: add_pair_v4i16_v4i32_zext: 1321; CHECK: // %bb.0: // %entry 1322; CHECK-NEXT: uaddl v0.4s, v0.4h, v1.4h 1323; CHECK-NEXT: addv s0, v0.4s 1324; CHECK-NEXT: fmov w0, s0 1325; CHECK-NEXT: ret 1326entry: 1327 %xx = zext <4 x i16> %x to <4 x i32> 1328 %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) 1329 %yy = zext <4 x i16> %y to <4 x i32> 1330 %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy) 1331 %z = add i32 %z1, %z2 1332 ret i32 %z 1333} 1334 1335define i32 @add_pair_v4i16_v4i32_sext(<4 x i16> %x, <4 x i16> %y) { 1336; CHECK-LABEL: add_pair_v4i16_v4i32_sext: 1337; CHECK: // %bb.0: // %entry 1338; CHECK-NEXT: saddl v0.4s, v0.4h, v1.4h 1339; CHECK-NEXT: addv s0, v0.4s 1340; CHECK-NEXT: fmov w0, s0 1341; CHECK-NEXT: ret 1342entry: 1343 %xx = sext <4 x i16> %x to <4 x i32> 1344 %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) 1345 %yy = sext <4 x i16> %y to <4 x i32> 1346 %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy) 1347 %z = add i32 %z1, %z2 1348 ret i32 %z 1349} 1350 1351define zeroext i16 @add_pair_v8i16_v8i16(<8 x i16> %x, <8 x i16> %y) { 1352; CHECK-LABEL: add_pair_v8i16_v8i16: 1353; CHECK: // %bb.0: // %entry 1354; CHECK-NEXT: addv h0, v0.8h 1355; CHECK-NEXT: addv h1, v1.8h 1356; CHECK-NEXT: fmov w8, s0 1357; CHECK-NEXT: fmov w9, s1 1358; CHECK-NEXT: add w8, w8, w9 1359; CHECK-NEXT: and w0, w8, #0xffff 1360; CHECK-NEXT: ret 1361entry: 1362 %z1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %x) 1363 %z2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %y) 1364 %z = add i16 %z1, %z2 1365 ret i16 %z 1366} 1367 1368define i64 @add_pair_v8i16_v8i64_zext(<8 x i16> %x, <8 x i16> %y) { 1369; CHECK-LABEL: add_pair_v8i16_v8i64_zext: 1370; CHECK: // %bb.0: // %entry 1371; CHECK-NEXT: ushll2 v2.4s, v0.8h, #0 1372; CHECK-NEXT: ushll v0.4s, v0.4h, #0 1373; CHECK-NEXT: ushll2 v3.4s, v1.8h, #0 1374; CHECK-NEXT: ushll v1.4s, v1.4h, #0 1375; CHECK-NEXT: uaddl2 v4.2d, v0.4s, v2.4s 1376; CHECK-NEXT: uaddl v0.2d, v0.2s, v2.2s 1377; CHECK-NEXT: uaddl2 v2.2d, v1.4s, v3.4s 1378; CHECK-NEXT: uaddl v1.2d, v1.2s, v3.2s 1379; CHECK-NEXT: add v0.2d, v0.2d, v4.2d 1380; CHECK-NEXT: add v1.2d, v1.2d, v2.2d 1381; CHECK-NEXT: add v0.2d, v0.2d, v1.2d 1382; CHECK-NEXT: addp d0, v0.2d 1383; CHECK-NEXT: fmov x0, d0 1384; CHECK-NEXT: ret 1385entry: 1386 %xx = zext <8 x i16> %x to <8 x i64> 1387 %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) 1388 %yy = zext <8 x i16> %y to <8 x i64> 1389 %z2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %yy) 1390 %z = add i64 %z1, %z2 1391 ret i64 %z 1392} 1393 1394define i64 @add_pair_v8i16_v8i64_sext(<8 x i16> %x, <8 x i16> %y) { 1395; CHECK-LABEL: add_pair_v8i16_v8i64_sext: 1396; CHECK: // %bb.0: // %entry 1397; CHECK-NEXT: sshll2 v2.4s, v0.8h, #0 1398; CHECK-NEXT: sshll v0.4s, v0.4h, #0 1399; CHECK-NEXT: sshll2 v3.4s, v1.8h, #0 1400; CHECK-NEXT: sshll v1.4s, v1.4h, #0 1401; CHECK-NEXT: saddl2 v4.2d, v0.4s, v2.4s 1402; CHECK-NEXT: saddl v0.2d, v0.2s, v2.2s 1403; CHECK-NEXT: saddl2 v2.2d, v1.4s, v3.4s 1404; CHECK-NEXT: saddl v1.2d, v1.2s, v3.2s 1405; CHECK-NEXT: add v0.2d, v0.2d, v4.2d 1406; CHECK-NEXT: add v1.2d, v1.2d, v2.2d 1407; CHECK-NEXT: add v0.2d, v0.2d, v1.2d 1408; CHECK-NEXT: addp d0, v0.2d 1409; CHECK-NEXT: fmov x0, d0 1410; CHECK-NEXT: ret 1411entry: 1412 %xx = sext <8 x i16> %x to <8 x i64> 1413 %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) 1414 %yy = sext <8 x i16> %y to <8 x i64> 1415 %z2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %yy) 1416 %z = add i64 %z1, %z2 1417 ret i64 %z 1418} 1419 1420define i64 @add_pair_v4i16_v4i64_zext(<4 x i16> %x, <4 x i16> %y) { 1421; CHECK-LABEL: add_pair_v4i16_v4i64_zext: 1422; CHECK: // %bb.0: // %entry 1423; CHECK-NEXT: ushll v0.4s, v0.4h, #0 1424; CHECK-NEXT: ushll v1.4s, v1.4h, #0 1425; CHECK-NEXT: uaddlp v0.2d, v0.4s 1426; CHECK-NEXT: uadalp v0.2d, v1.4s 1427; CHECK-NEXT: addp d0, v0.2d 1428; CHECK-NEXT: fmov x0, d0 1429; CHECK-NEXT: ret 1430entry: 1431 %xx = zext <4 x i16> %x to <4 x i64> 1432 %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 1433 %yy = zext <4 x i16> %y to <4 x i64> 1434 %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy) 1435 %z = add i64 %z1, %z2 1436 ret i64 %z 1437} 1438 1439define i64 @add_pair_v4i16_v4i64_sext(<4 x i16> %x, <4 x i16> %y) { 1440; CHECK-LABEL: add_pair_v4i16_v4i64_sext: 1441; CHECK: // %bb.0: // %entry 1442; CHECK-NEXT: sshll v0.4s, v0.4h, #0 1443; CHECK-NEXT: sshll v1.4s, v1.4h, #0 1444; CHECK-NEXT: saddlp v0.2d, v0.4s 1445; CHECK-NEXT: sadalp v0.2d, v1.4s 1446; CHECK-NEXT: addp d0, v0.2d 1447; CHECK-NEXT: fmov x0, d0 1448; CHECK-NEXT: ret 1449entry: 1450 %xx = sext <4 x i16> %x to <4 x i64> 1451 %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 1452 %yy = sext <4 x i16> %y to <4 x i64> 1453 %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy) 1454 %z = add i64 %z1, %z2 1455 ret i64 %z 1456} 1457 1458define i64 @add_pair_v2i16_v2i64_zext(<2 x i16> %x, <2 x i16> %y) { 1459; CHECK-LABEL: add_pair_v2i16_v2i64_zext: 1460; CHECK: // %bb.0: // %entry 1461; CHECK-NEXT: movi d2, #0x00ffff0000ffff 1462; CHECK-NEXT: and v0.8b, v0.8b, v2.8b 1463; CHECK-NEXT: and v1.8b, v1.8b, v2.8b 1464; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s 1465; CHECK-NEXT: addp d0, v0.2d 1466; CHECK-NEXT: fmov x0, d0 1467; CHECK-NEXT: ret 1468entry: 1469 %xx = zext <2 x i16> %x to <2 x i64> 1470 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 1471 %yy = zext <2 x i16> %y to <2 x i64> 1472 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy) 1473 %z = add i64 %z1, %z2 1474 ret i64 %z 1475} 1476 1477define i64 @add_pair_v2i16_v2i64_sext(<2 x i16> %x, <2 x i16> %y) { 1478; CHECK-LABEL: add_pair_v2i16_v2i64_sext: 1479; CHECK: // %bb.0: // %entry 1480; CHECK-NEXT: ushll v0.2d, v0.2s, #0 1481; CHECK-NEXT: ushll v1.2d, v1.2s, #0 1482; CHECK-NEXT: shl v0.2d, v0.2d, #48 1483; CHECK-NEXT: shl v1.2d, v1.2d, #48 1484; CHECK-NEXT: sshr v0.2d, v0.2d, #48 1485; CHECK-NEXT: ssra v0.2d, v1.2d, #48 1486; CHECK-NEXT: addp d0, v0.2d 1487; CHECK-NEXT: fmov x0, d0 1488; CHECK-NEXT: ret 1489entry: 1490 %xx = sext <2 x i16> %x to <2 x i64> 1491 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 1492 %yy = sext <2 x i16> %y to <2 x i64> 1493 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy) 1494 %z = add i64 %z1, %z2 1495 ret i64 %z 1496} 1497 1498define i32 @add_pair_v16i8_v16i32_zext(<16 x i8> %x, <16 x i8> %y) { 1499; CHECK-BASE-LABEL: add_pair_v16i8_v16i32_zext: 1500; CHECK-BASE: // %bb.0: // %entry 1501; CHECK-BASE-NEXT: ushll2 v2.8h, v0.16b, #0 1502; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0 1503; CHECK-BASE-NEXT: ushll2 v3.8h, v1.16b, #0 1504; CHECK-BASE-NEXT: ushll v1.8h, v1.8b, #0 1505; CHECK-BASE-NEXT: uaddl2 v4.4s, v0.8h, v2.8h 1506; CHECK-BASE-NEXT: uaddl v0.4s, v0.4h, v2.4h 1507; CHECK-BASE-NEXT: uaddl2 v2.4s, v1.8h, v3.8h 1508; CHECK-BASE-NEXT: uaddl v1.4s, v1.4h, v3.4h 1509; CHECK-BASE-NEXT: add v0.4s, v0.4s, v4.4s 1510; CHECK-BASE-NEXT: add v1.4s, v1.4s, v2.4s 1511; CHECK-BASE-NEXT: add v0.4s, v0.4s, v1.4s 1512; CHECK-BASE-NEXT: addv s0, v0.4s 1513; CHECK-BASE-NEXT: fmov w0, s0 1514; CHECK-BASE-NEXT: ret 1515; 1516; CHECK-DOT-LABEL: add_pair_v16i8_v16i32_zext: 1517; CHECK-DOT: // %bb.0: // %entry 1518; CHECK-DOT-NEXT: movi v2.16b, #1 1519; CHECK-DOT-NEXT: movi v3.2d, #0000000000000000 1520; CHECK-DOT-NEXT: udot v3.4s, v1.16b, v2.16b 1521; CHECK-DOT-NEXT: udot v3.4s, v0.16b, v2.16b 1522; CHECK-DOT-NEXT: addv s0, v3.4s 1523; CHECK-DOT-NEXT: fmov w0, s0 1524; CHECK-DOT-NEXT: ret 1525entry: 1526 %xx = zext <16 x i8> %x to <16 x i32> 1527 %z1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx) 1528 %yy = zext <16 x i8> %y to <16 x i32> 1529 %z2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %yy) 1530 %z = add i32 %z1, %z2 1531 ret i32 %z 1532} 1533 1534define i32 @add_pair_v16i8_v16i32_sext(<16 x i8> %x, <16 x i8> %y) { 1535; CHECK-BASE-LABEL: add_pair_v16i8_v16i32_sext: 1536; CHECK-BASE: // %bb.0: // %entry 1537; CHECK-BASE-NEXT: sshll2 v2.8h, v0.16b, #0 1538; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0 1539; CHECK-BASE-NEXT: sshll2 v3.8h, v1.16b, #0 1540; CHECK-BASE-NEXT: sshll v1.8h, v1.8b, #0 1541; CHECK-BASE-NEXT: saddl2 v4.4s, v0.8h, v2.8h 1542; CHECK-BASE-NEXT: saddl v0.4s, v0.4h, v2.4h 1543; CHECK-BASE-NEXT: saddl2 v2.4s, v1.8h, v3.8h 1544; CHECK-BASE-NEXT: saddl v1.4s, v1.4h, v3.4h 1545; CHECK-BASE-NEXT: add v0.4s, v0.4s, v4.4s 1546; CHECK-BASE-NEXT: add v1.4s, v1.4s, v2.4s 1547; CHECK-BASE-NEXT: add v0.4s, v0.4s, v1.4s 1548; CHECK-BASE-NEXT: addv s0, v0.4s 1549; CHECK-BASE-NEXT: fmov w0, s0 1550; CHECK-BASE-NEXT: ret 1551; 1552; CHECK-DOT-LABEL: add_pair_v16i8_v16i32_sext: 1553; CHECK-DOT: // %bb.0: // %entry 1554; CHECK-DOT-NEXT: movi v2.16b, #1 1555; CHECK-DOT-NEXT: movi v3.2d, #0000000000000000 1556; CHECK-DOT-NEXT: sdot v3.4s, v1.16b, v2.16b 1557; CHECK-DOT-NEXT: sdot v3.4s, v0.16b, v2.16b 1558; CHECK-DOT-NEXT: addv s0, v3.4s 1559; CHECK-DOT-NEXT: fmov w0, s0 1560; CHECK-DOT-NEXT: ret 1561entry: 1562 %xx = sext <16 x i8> %x to <16 x i32> 1563 %z1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx) 1564 %yy = sext <16 x i8> %y to <16 x i32> 1565 %z2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %yy) 1566 %z = add i32 %z1, %z2 1567 ret i32 %z 1568} 1569 1570define i32 @add_pair_v8i8_v8i32_zext(<8 x i8> %x, <8 x i8> %y) { 1571; CHECK-BASE-LABEL: add_pair_v8i8_v8i32_zext: 1572; CHECK-BASE: // %bb.0: // %entry 1573; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0 1574; CHECK-BASE-NEXT: ushll v1.8h, v1.8b, #0 1575; CHECK-BASE-NEXT: uaddlp v0.4s, v0.8h 1576; CHECK-BASE-NEXT: uadalp v0.4s, v1.8h 1577; CHECK-BASE-NEXT: addv s0, v0.4s 1578; CHECK-BASE-NEXT: fmov w0, s0 1579; CHECK-BASE-NEXT: ret 1580; 1581; CHECK-DOT-LABEL: add_pair_v8i8_v8i32_zext: 1582; CHECK-DOT: // %bb.0: // %entry 1583; CHECK-DOT-NEXT: movi v2.8b, #1 1584; CHECK-DOT-NEXT: movi v3.2d, #0000000000000000 1585; CHECK-DOT-NEXT: udot v3.2s, v1.8b, v2.8b 1586; CHECK-DOT-NEXT: udot v3.2s, v0.8b, v2.8b 1587; CHECK-DOT-NEXT: addp v0.2s, v3.2s, v3.2s 1588; CHECK-DOT-NEXT: fmov w0, s0 1589; CHECK-DOT-NEXT: ret 1590entry: 1591 %xx = zext <8 x i8> %x to <8 x i32> 1592 %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) 1593 %yy = zext <8 x i8> %y to <8 x i32> 1594 %z2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %yy) 1595 %z = add i32 %z1, %z2 1596 ret i32 %z 1597} 1598 1599define i32 @add_pair_v8i8_v8i32_sext(<8 x i8> %x, <8 x i8> %y) { 1600; CHECK-BASE-LABEL: add_pair_v8i8_v8i32_sext: 1601; CHECK-BASE: // %bb.0: // %entry 1602; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0 1603; CHECK-BASE-NEXT: sshll v1.8h, v1.8b, #0 1604; CHECK-BASE-NEXT: saddlp v0.4s, v0.8h 1605; CHECK-BASE-NEXT: sadalp v0.4s, v1.8h 1606; CHECK-BASE-NEXT: addv s0, v0.4s 1607; CHECK-BASE-NEXT: fmov w0, s0 1608; CHECK-BASE-NEXT: ret 1609; 1610; CHECK-DOT-LABEL: add_pair_v8i8_v8i32_sext: 1611; CHECK-DOT: // %bb.0: // %entry 1612; CHECK-DOT-NEXT: movi v2.8b, #1 1613; CHECK-DOT-NEXT: movi v3.2d, #0000000000000000 1614; CHECK-DOT-NEXT: sdot v3.2s, v1.8b, v2.8b 1615; CHECK-DOT-NEXT: sdot v3.2s, v0.8b, v2.8b 1616; CHECK-DOT-NEXT: addp v0.2s, v3.2s, v3.2s 1617; CHECK-DOT-NEXT: fmov w0, s0 1618; CHECK-DOT-NEXT: ret 1619entry: 1620 %xx = sext <8 x i8> %x to <8 x i32> 1621 %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) 1622 %yy = sext <8 x i8> %y to <8 x i32> 1623 %z2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %yy) 1624 %z = add i32 %z1, %z2 1625 ret i32 %z 1626} 1627 1628define i32 @add_pair_v4i8_v4i32_zext(<4 x i8> %x, <4 x i8> %y) { 1629; CHECK-LABEL: add_pair_v4i8_v4i32_zext: 1630; CHECK: // %bb.0: // %entry 1631; CHECK-NEXT: bic v0.4h, #255, lsl #8 1632; CHECK-NEXT: bic v1.4h, #255, lsl #8 1633; CHECK-NEXT: uaddl v0.4s, v0.4h, v1.4h 1634; CHECK-NEXT: addv s0, v0.4s 1635; CHECK-NEXT: fmov w0, s0 1636; CHECK-NEXT: ret 1637entry: 1638 %xx = zext <4 x i8> %x to <4 x i32> 1639 %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) 1640 %yy = zext <4 x i8> %y to <4 x i32> 1641 %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy) 1642 %z = add i32 %z1, %z2 1643 ret i32 %z 1644} 1645 1646define i32 @add_pair_v4i8_v4i32_sext(<4 x i8> %x, <4 x i8> %y) { 1647; CHECK-LABEL: add_pair_v4i8_v4i32_sext: 1648; CHECK: // %bb.0: // %entry 1649; CHECK-NEXT: ushll v0.4s, v0.4h, #0 1650; CHECK-NEXT: ushll v1.4s, v1.4h, #0 1651; CHECK-NEXT: shl v0.4s, v0.4s, #24 1652; CHECK-NEXT: shl v1.4s, v1.4s, #24 1653; CHECK-NEXT: sshr v0.4s, v0.4s, #24 1654; CHECK-NEXT: ssra v0.4s, v1.4s, #24 1655; CHECK-NEXT: addv s0, v0.4s 1656; CHECK-NEXT: fmov w0, s0 1657; CHECK-NEXT: ret 1658entry: 1659 %xx = sext <4 x i8> %x to <4 x i32> 1660 %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) 1661 %yy = sext <4 x i8> %y to <4 x i32> 1662 %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy) 1663 %z = add i32 %z1, %z2 1664 ret i32 %z 1665} 1666 1667define zeroext i16 @add_pair_v16i8_v16i16_zext(<16 x i8> %x, <16 x i8> %y) { 1668; CHECK-LABEL: add_pair_v16i8_v16i16_zext: 1669; CHECK: // %bb.0: // %entry 1670; CHECK-NEXT: uaddlv h0, v0.16b 1671; CHECK-NEXT: uaddlv h1, v1.16b 1672; CHECK-NEXT: fmov w8, s0 1673; CHECK-NEXT: fmov w9, s1 1674; CHECK-NEXT: add w8, w8, w9 1675; CHECK-NEXT: and w0, w8, #0xffff 1676; CHECK-NEXT: ret 1677entry: 1678 %xx = zext <16 x i8> %x to <16 x i16> 1679 %z1 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx) 1680 %yy = zext <16 x i8> %y to <16 x i16> 1681 %z2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %yy) 1682 %z = add i16 %z1, %z2 1683 ret i16 %z 1684} 1685 1686define signext i16 @add_pair_v16i8_v16i16_sext(<16 x i8> %x, <16 x i8> %y) { 1687; CHECK-LABEL: add_pair_v16i8_v16i16_sext: 1688; CHECK: // %bb.0: // %entry 1689; CHECK-NEXT: saddlv h0, v0.16b 1690; CHECK-NEXT: saddlv h1, v1.16b 1691; CHECK-NEXT: fmov w8, s0 1692; CHECK-NEXT: fmov w9, s1 1693; CHECK-NEXT: add w8, w8, w9 1694; CHECK-NEXT: sxth w0, w8 1695; CHECK-NEXT: ret 1696entry: 1697 %xx = sext <16 x i8> %x to <16 x i16> 1698 %z1 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx) 1699 %yy = sext <16 x i8> %y to <16 x i16> 1700 %z2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %yy) 1701 %z = add i16 %z1, %z2 1702 ret i16 %z 1703} 1704 1705define zeroext i16 @add_pair_v8i8_v8i16_zext(<8 x i8> %x, <8 x i8> %y) { 1706; CHECK-LABEL: add_pair_v8i8_v8i16_zext: 1707; CHECK: // %bb.0: // %entry 1708; CHECK-NEXT: ushll v0.8h, v0.8b, #0 1709; CHECK-NEXT: ushll v1.8h, v1.8b, #0 1710; CHECK-NEXT: addv h0, v0.8h 1711; CHECK-NEXT: addv h1, v1.8h 1712; CHECK-NEXT: fmov w8, s0 1713; CHECK-NEXT: fmov w9, s1 1714; CHECK-NEXT: add w8, w8, w9 1715; CHECK-NEXT: and w0, w8, #0xffff 1716; CHECK-NEXT: ret 1717entry: 1718 %xx = zext <8 x i8> %x to <8 x i16> 1719 %z1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx) 1720 %yy = zext <8 x i8> %y to <8 x i16> 1721 %z2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %yy) 1722 %z = add i16 %z1, %z2 1723 ret i16 %z 1724} 1725 1726define signext i16 @add_pair_v8i8_v8i16_sext(<8 x i8> %x, <8 x i8> %y) { 1727; CHECK-LABEL: add_pair_v8i8_v8i16_sext: 1728; CHECK: // %bb.0: // %entry 1729; CHECK-NEXT: sshll v0.8h, v0.8b, #0 1730; CHECK-NEXT: sshll v1.8h, v1.8b, #0 1731; CHECK-NEXT: addv h0, v0.8h 1732; CHECK-NEXT: addv h1, v1.8h 1733; CHECK-NEXT: fmov w8, s0 1734; CHECK-NEXT: fmov w9, s1 1735; CHECK-NEXT: add w8, w8, w9 1736; CHECK-NEXT: sxth w0, w8 1737; CHECK-NEXT: ret 1738entry: 1739 %xx = sext <8 x i8> %x to <8 x i16> 1740 %z1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx) 1741 %yy = sext <8 x i8> %y to <8 x i16> 1742 %z2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %yy) 1743 %z = add i16 %z1, %z2 1744 ret i16 %z 1745} 1746 1747define zeroext i8 @add_pair_v16i8_v16i8(<16 x i8> %x, <16 x i8> %y) { 1748; CHECK-LABEL: add_pair_v16i8_v16i8: 1749; CHECK: // %bb.0: // %entry 1750; CHECK-NEXT: addv b0, v0.16b 1751; CHECK-NEXT: addv b1, v1.16b 1752; CHECK-NEXT: fmov w8, s0 1753; CHECK-NEXT: fmov w9, s1 1754; CHECK-NEXT: add w8, w8, w9 1755; CHECK-NEXT: and w0, w8, #0xff 1756; CHECK-NEXT: ret 1757entry: 1758 %z1 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %x) 1759 %z2 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %y) 1760 %z = add i8 %z1, %z2 1761 ret i8 %z 1762} 1763 1764define i64 @add_pair_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y) { 1765; CHECK-LABEL: add_pair_v16i8_v16i64_zext: 1766; CHECK: // %bb.0: // %entry 1767; CHECK-NEXT: ushll2 v2.8h, v0.16b, #0 1768; CHECK-NEXT: ushll v0.8h, v0.8b, #0 1769; CHECK-NEXT: ushll v3.4s, v2.4h, #0 1770; CHECK-NEXT: ushll2 v2.4s, v2.8h, #0 1771; CHECK-NEXT: ushll2 v4.4s, v0.8h, #0 1772; CHECK-NEXT: ushll v0.4s, v0.4h, #0 1773; CHECK-NEXT: uaddl2 v5.2d, v4.4s, v2.4s 1774; CHECK-NEXT: uaddl2 v6.2d, v0.4s, v3.4s 1775; CHECK-NEXT: ushll2 v7.8h, v1.16b, #0 1776; CHECK-NEXT: ushll v1.8h, v1.8b, #0 1777; CHECK-NEXT: uaddl v2.2d, v4.2s, v2.2s 1778; CHECK-NEXT: add v4.2d, v6.2d, v5.2d 1779; CHECK-NEXT: uaddl v0.2d, v0.2s, v3.2s 1780; CHECK-NEXT: ushll v3.4s, v7.4h, #0 1781; CHECK-NEXT: ushll2 v5.4s, v7.8h, #0 1782; CHECK-NEXT: ushll2 v6.4s, v1.8h, #0 1783; CHECK-NEXT: ushll v1.4s, v1.4h, #0 1784; CHECK-NEXT: uaddl2 v7.2d, v6.4s, v5.4s 1785; CHECK-NEXT: uaddl v5.2d, v6.2s, v5.2s 1786; CHECK-NEXT: uaddl2 v6.2d, v1.4s, v3.4s 1787; CHECK-NEXT: uaddl v1.2d, v1.2s, v3.2s 1788; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 1789; CHECK-NEXT: add v2.2d, v6.2d, v7.2d 1790; CHECK-NEXT: add v1.2d, v1.2d, v5.2d 1791; CHECK-NEXT: add v0.2d, v0.2d, v4.2d 1792; CHECK-NEXT: add v1.2d, v1.2d, v2.2d 1793; CHECK-NEXT: add v0.2d, v0.2d, v1.2d 1794; CHECK-NEXT: addp d0, v0.2d 1795; CHECK-NEXT: fmov x0, d0 1796; CHECK-NEXT: ret 1797entry: 1798 %xx = zext <16 x i8> %x to <16 x i64> 1799 %z1 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx) 1800 %yy = zext <16 x i8> %y to <16 x i64> 1801 %z2 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %yy) 1802 %z = add i64 %z1, %z2 1803 ret i64 %z 1804} 1805 1806define i64 @add_pair_v16i8_v16i64_sext(<16 x i8> %x, <16 x i8> %y) { 1807; CHECK-LABEL: add_pair_v16i8_v16i64_sext: 1808; CHECK: // %bb.0: // %entry 1809; CHECK-NEXT: sshll2 v2.8h, v0.16b, #0 1810; CHECK-NEXT: sshll v0.8h, v0.8b, #0 1811; CHECK-NEXT: sshll v3.4s, v2.4h, #0 1812; CHECK-NEXT: sshll2 v2.4s, v2.8h, #0 1813; CHECK-NEXT: sshll2 v4.4s, v0.8h, #0 1814; CHECK-NEXT: sshll v0.4s, v0.4h, #0 1815; CHECK-NEXT: saddl2 v5.2d, v4.4s, v2.4s 1816; CHECK-NEXT: saddl2 v6.2d, v0.4s, v3.4s 1817; CHECK-NEXT: sshll2 v7.8h, v1.16b, #0 1818; CHECK-NEXT: sshll v1.8h, v1.8b, #0 1819; CHECK-NEXT: saddl v2.2d, v4.2s, v2.2s 1820; CHECK-NEXT: add v4.2d, v6.2d, v5.2d 1821; CHECK-NEXT: saddl v0.2d, v0.2s, v3.2s 1822; CHECK-NEXT: sshll v3.4s, v7.4h, #0 1823; CHECK-NEXT: sshll2 v5.4s, v7.8h, #0 1824; CHECK-NEXT: sshll2 v6.4s, v1.8h, #0 1825; CHECK-NEXT: sshll v1.4s, v1.4h, #0 1826; CHECK-NEXT: saddl2 v7.2d, v6.4s, v5.4s 1827; CHECK-NEXT: saddl v5.2d, v6.2s, v5.2s 1828; CHECK-NEXT: saddl2 v6.2d, v1.4s, v3.4s 1829; CHECK-NEXT: saddl v1.2d, v1.2s, v3.2s 1830; CHECK-NEXT: add v0.2d, v0.2d, v2.2d 1831; CHECK-NEXT: add v2.2d, v6.2d, v7.2d 1832; CHECK-NEXT: add v1.2d, v1.2d, v5.2d 1833; CHECK-NEXT: add v0.2d, v0.2d, v4.2d 1834; CHECK-NEXT: add v1.2d, v1.2d, v2.2d 1835; CHECK-NEXT: add v0.2d, v0.2d, v1.2d 1836; CHECK-NEXT: addp d0, v0.2d 1837; CHECK-NEXT: fmov x0, d0 1838; CHECK-NEXT: ret 1839entry: 1840 %xx = sext <16 x i8> %x to <16 x i64> 1841 %z1 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx) 1842 %yy = sext <16 x i8> %y to <16 x i64> 1843 %z2 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %yy) 1844 %z = add i64 %z1, %z2 1845 ret i64 %z 1846} 1847 1848define i64 @add_pair_v8i8_v8i64_zext(<8 x i8> %x, <8 x i8> %y) { 1849; CHECK-LABEL: add_pair_v8i8_v8i64_zext: 1850; CHECK: // %bb.0: // %entry 1851; CHECK-NEXT: ushll v0.8h, v0.8b, #0 1852; CHECK-NEXT: ushll v1.8h, v1.8b, #0 1853; CHECK-NEXT: ushll2 v2.4s, v0.8h, #0 1854; CHECK-NEXT: ushll v0.4s, v0.4h, #0 1855; CHECK-NEXT: ushll2 v3.4s, v1.8h, #0 1856; CHECK-NEXT: ushll v1.4s, v1.4h, #0 1857; CHECK-NEXT: uaddl2 v4.2d, v0.4s, v2.4s 1858; CHECK-NEXT: uaddl v0.2d, v0.2s, v2.2s 1859; CHECK-NEXT: uaddl2 v2.2d, v1.4s, v3.4s 1860; CHECK-NEXT: uaddl v1.2d, v1.2s, v3.2s 1861; CHECK-NEXT: add v0.2d, v0.2d, v4.2d 1862; CHECK-NEXT: add v1.2d, v1.2d, v2.2d 1863; CHECK-NEXT: add v0.2d, v0.2d, v1.2d 1864; CHECK-NEXT: addp d0, v0.2d 1865; CHECK-NEXT: fmov x0, d0 1866; CHECK-NEXT: ret 1867entry: 1868 %xx = zext <8 x i8> %x to <8 x i64> 1869 %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) 1870 %yy = zext <8 x i8> %y to <8 x i64> 1871 %z2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %yy) 1872 %z = add i64 %z1, %z2 1873 ret i64 %z 1874} 1875 1876define i64 @add_pair_v8i8_v8i64_sext(<8 x i8> %x, <8 x i8> %y) { 1877; CHECK-LABEL: add_pair_v8i8_v8i64_sext: 1878; CHECK: // %bb.0: // %entry 1879; CHECK-NEXT: sshll v0.8h, v0.8b, #0 1880; CHECK-NEXT: sshll v1.8h, v1.8b, #0 1881; CHECK-NEXT: sshll2 v2.4s, v0.8h, #0 1882; CHECK-NEXT: sshll v0.4s, v0.4h, #0 1883; CHECK-NEXT: sshll2 v3.4s, v1.8h, #0 1884; CHECK-NEXT: sshll v1.4s, v1.4h, #0 1885; CHECK-NEXT: saddl2 v4.2d, v0.4s, v2.4s 1886; CHECK-NEXT: saddl v0.2d, v0.2s, v2.2s 1887; CHECK-NEXT: saddl2 v2.2d, v1.4s, v3.4s 1888; CHECK-NEXT: saddl v1.2d, v1.2s, v3.2s 1889; CHECK-NEXT: add v0.2d, v0.2d, v4.2d 1890; CHECK-NEXT: add v1.2d, v1.2d, v2.2d 1891; CHECK-NEXT: add v0.2d, v0.2d, v1.2d 1892; CHECK-NEXT: addp d0, v0.2d 1893; CHECK-NEXT: fmov x0, d0 1894; CHECK-NEXT: ret 1895entry: 1896 %xx = sext <8 x i8> %x to <8 x i64> 1897 %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) 1898 %yy = sext <8 x i8> %y to <8 x i64> 1899 %z2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %yy) 1900 %z = add i64 %z1, %z2 1901 ret i64 %z 1902} 1903 1904define i64 @add_pair_v4i8_v4i64_zext(<4 x i8> %x, <4 x i8> %y) { 1905; CHECK-LABEL: add_pair_v4i8_v4i64_zext: 1906; CHECK: // %bb.0: // %entry 1907; CHECK-NEXT: bic v0.4h, #255, lsl #8 1908; CHECK-NEXT: bic v1.4h, #255, lsl #8 1909; CHECK-NEXT: ushll v0.4s, v0.4h, #0 1910; CHECK-NEXT: ushll v1.4s, v1.4h, #0 1911; CHECK-NEXT: uaddlp v0.2d, v0.4s 1912; CHECK-NEXT: uadalp v0.2d, v1.4s 1913; CHECK-NEXT: addp d0, v0.2d 1914; CHECK-NEXT: fmov x0, d0 1915; CHECK-NEXT: ret 1916entry: 1917 %xx = zext <4 x i8> %x to <4 x i64> 1918 %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 1919 %yy = zext <4 x i8> %y to <4 x i64> 1920 %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy) 1921 %z = add i64 %z1, %z2 1922 ret i64 %z 1923} 1924 1925define i64 @add_pair_v4i8_v4i64_sext(<4 x i8> %x, <4 x i8> %y) { 1926; CHECK-LABEL: add_pair_v4i8_v4i64_sext: 1927; CHECK: // %bb.0: // %entry 1928; CHECK-NEXT: ushll v1.4s, v1.4h, #0 1929; CHECK-NEXT: ushll v0.4s, v0.4h, #0 1930; CHECK-NEXT: ushll v2.2d, v1.2s, #0 1931; CHECK-NEXT: ushll v3.2d, v0.2s, #0 1932; CHECK-NEXT: ushll2 v1.2d, v1.4s, #0 1933; CHECK-NEXT: ushll2 v0.2d, v0.4s, #0 1934; CHECK-NEXT: shl v3.2d, v3.2d, #56 1935; CHECK-NEXT: shl v2.2d, v2.2d, #56 1936; CHECK-NEXT: shl v0.2d, v0.2d, #56 1937; CHECK-NEXT: sshr v3.2d, v3.2d, #56 1938; CHECK-NEXT: shl v1.2d, v1.2d, #56 1939; CHECK-NEXT: sshr v2.2d, v2.2d, #56 1940; CHECK-NEXT: ssra v3.2d, v0.2d, #56 1941; CHECK-NEXT: ssra v2.2d, v1.2d, #56 1942; CHECK-NEXT: add v0.2d, v3.2d, v2.2d 1943; CHECK-NEXT: addp d0, v0.2d 1944; CHECK-NEXT: fmov x0, d0 1945; CHECK-NEXT: ret 1946entry: 1947 %xx = sext <4 x i8> %x to <4 x i64> 1948 %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) 1949 %yy = sext <4 x i8> %y to <4 x i64> 1950 %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy) 1951 %z = add i64 %z1, %z2 1952 ret i64 %z 1953} 1954 1955define i64 @add_pair_v2i8_v2i64_zext(<2 x i8> %x, <2 x i8> %y) { 1956; CHECK-LABEL: add_pair_v2i8_v2i64_zext: 1957; CHECK: // %bb.0: // %entry 1958; CHECK-NEXT: movi d2, #0x0000ff000000ff 1959; CHECK-NEXT: and v0.8b, v0.8b, v2.8b 1960; CHECK-NEXT: and v1.8b, v1.8b, v2.8b 1961; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s 1962; CHECK-NEXT: addp d0, v0.2d 1963; CHECK-NEXT: fmov x0, d0 1964; CHECK-NEXT: ret 1965entry: 1966 %xx = zext <2 x i8> %x to <2 x i64> 1967 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 1968 %yy = zext <2 x i8> %y to <2 x i64> 1969 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy) 1970 %z = add i64 %z1, %z2 1971 ret i64 %z 1972} 1973 1974define i64 @add_pair_v2i8_v2i64_sext(<2 x i8> %x, <2 x i8> %y) { 1975; CHECK-LABEL: add_pair_v2i8_v2i64_sext: 1976; CHECK: // %bb.0: // %entry 1977; CHECK-NEXT: ushll v0.2d, v0.2s, #0 1978; CHECK-NEXT: ushll v1.2d, v1.2s, #0 1979; CHECK-NEXT: shl v0.2d, v0.2d, #56 1980; CHECK-NEXT: shl v1.2d, v1.2d, #56 1981; CHECK-NEXT: sshr v0.2d, v0.2d, #56 1982; CHECK-NEXT: ssra v0.2d, v1.2d, #56 1983; CHECK-NEXT: addp d0, v0.2d 1984; CHECK-NEXT: fmov x0, d0 1985; CHECK-NEXT: ret 1986entry: 1987 %xx = sext <2 x i8> %x to <2 x i64> 1988 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) 1989 %yy = sext <2 x i8> %y to <2 x i64> 1990 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy) 1991 %z = add i64 %z1, %z2 1992 ret i64 %z 1993} 1994 1995define i64 @add_pair_v2i64_v2i64(<2 x i64> %x, <2 x i64> %y) { 1996; CHECK-LABEL: add_pair_v2i64_v2i64: 1997; CHECK: // %bb.0: // %entry 1998; CHECK-NEXT: add v0.2d, v0.2d, v1.2d 1999; CHECK-NEXT: addp d0, v0.2d 2000; CHECK-NEXT: fmov x0, d0 2001; CHECK-NEXT: ret 2002entry: 2003 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %x) 2004 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %y) 2005 %z = add i64 %z1, %z2 2006 ret i64 %z 2007} 2008 2009declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>) 2010declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>) 2011declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>) 2012declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>) 2013declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>) 2014declare i64 @llvm.vector.reduce.add.v16i64(<16 x i64>) 2015declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>) 2016declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>) 2017declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>) 2018declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>) 2019