1 //===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// This file implements the targeting of the Machinelegalizer class for 10 /// AArch64. 11 /// \todo This should be generated by TableGen. 12 //===----------------------------------------------------------------------===// 13 14 #include "AArch64LegalizerInfo.h" 15 #include "AArch64Subtarget.h" 16 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" 17 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" 18 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 19 #include "llvm/CodeGen/GlobalISel/Utils.h" 20 #include "llvm/CodeGen/MachineInstr.h" 21 #include "llvm/CodeGen/MachineRegisterInfo.h" 22 #include "llvm/CodeGen/TargetOpcodes.h" 23 #include "llvm/CodeGen/ValueTypes.h" 24 #include "llvm/IR/DerivedTypes.h" 25 #include "llvm/IR/IntrinsicsAArch64.h" 26 #include "llvm/IR/Type.h" 27 #include "llvm/Support/MathExtras.h" 28 #include <initializer_list> 29 30 #define DEBUG_TYPE "aarch64-legalinfo" 31 32 using namespace llvm; 33 using namespace LegalizeActions; 34 using namespace LegalizeMutations; 35 using namespace LegalityPredicates; 36 37 AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) 38 : ST(&ST) { 39 using namespace TargetOpcode; 40 const LLT p0 = LLT::pointer(0, 64); 41 const LLT s1 = LLT::scalar(1); 42 const LLT s8 = LLT::scalar(8); 43 const LLT s16 = LLT::scalar(16); 44 const LLT s32 = LLT::scalar(32); 45 const LLT s64 = LLT::scalar(64); 46 const LLT s128 = LLT::scalar(128); 47 const LLT s256 = LLT::scalar(256); 48 const LLT s512 = LLT::scalar(512); 49 const LLT v16s8 = LLT::vector(16, 8); 50 const LLT v8s8 = LLT::vector(8, 8); 51 const LLT v4s8 = LLT::vector(4, 8); 52 const LLT v8s16 = LLT::vector(8, 16); 53 const LLT v4s16 = LLT::vector(4, 16); 54 const LLT v2s16 = LLT::vector(2, 16); 55 const LLT v2s32 = LLT::vector(2, 32); 56 const LLT v4s32 = LLT::vector(4, 32); 57 const LLT v2s64 = LLT::vector(2, 64); 58 const LLT v2p0 = LLT::vector(2, p0); 59 60 std::initializer_list<LLT> PackedVectorAllTypeList = {/* Begin 128bit types */ 61 v16s8, v8s16, v4s32, 62 v2s64, v2p0, 63 /* End 128bit types */ 64 /* Begin 64bit types */ 65 v8s8, v4s16, v2s32}; 66 67 const TargetMachine &TM = ST.getTargetLowering()->getTargetMachine(); 68 69 // FIXME: support subtargets which have neon/fp-armv8 disabled. 70 if (!ST.hasNEON() || !ST.hasFPARMv8()) { 71 computeTables(); 72 return; 73 } 74 75 // Some instructions only support s16 if the subtarget has full 16-bit FP 76 // support. 77 const bool HasFP16 = ST.hasFullFP16(); 78 const LLT &MinFPScalar = HasFP16 ? s16 : s32; 79 80 getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE}) 81 .legalFor({p0, s1, s8, s16, s32, s64}) 82 .legalFor(PackedVectorAllTypeList) 83 .clampScalar(0, s1, s64) 84 .widenScalarToNextPow2(0, 8) 85 .fewerElementsIf( 86 [=](const LegalityQuery &Query) { 87 return Query.Types[0].isVector() && 88 (Query.Types[0].getElementType() != s64 || 89 Query.Types[0].getNumElements() != 2); 90 }, 91 [=](const LegalityQuery &Query) { 92 LLT EltTy = Query.Types[0].getElementType(); 93 if (EltTy == s64) 94 return std::make_pair(0, LLT::vector(2, 64)); 95 return std::make_pair(0, EltTy); 96 }); 97 98 getActionDefinitionsBuilder(G_PHI).legalFor({p0, s16, s32, s64}) 99 .legalFor(PackedVectorAllTypeList) 100 .clampScalar(0, s16, s64) 101 .widenScalarToNextPow2(0); 102 103 getActionDefinitionsBuilder(G_BSWAP) 104 .legalFor({s32, s64, v4s32, v2s32, v2s64}) 105 .clampScalar(0, s32, s64) 106 .widenScalarToNextPow2(0); 107 108 getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR}) 109 .legalFor({s32, s64, v2s32, v4s32, v4s16, v8s16, v16s8, v8s8}) 110 .scalarizeIf( 111 [=](const LegalityQuery &Query) { 112 return Query.Opcode == G_MUL && Query.Types[0] == v2s64; 113 }, 114 0) 115 .legalFor({v2s64}) 116 .clampScalar(0, s32, s64) 117 .widenScalarToNextPow2(0) 118 .clampNumElements(0, v2s32, v4s32) 119 .clampNumElements(0, v2s64, v2s64) 120 .moreElementsToNextPow2(0); 121 122 getActionDefinitionsBuilder({G_SHL, G_ASHR, G_LSHR}) 123 .customIf([=](const LegalityQuery &Query) { 124 const auto &SrcTy = Query.Types[0]; 125 const auto &AmtTy = Query.Types[1]; 126 return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 && 127 AmtTy.getSizeInBits() == 32; 128 }) 129 .legalFor({ 130 {s32, s32}, 131 {s32, s64}, 132 {s64, s64}, 133 {v8s8, v8s8}, 134 {v16s8, v16s8}, 135 {v4s16, v4s16}, 136 {v8s16, v8s16}, 137 {v2s32, v2s32}, 138 {v4s32, v4s32}, 139 {v2s64, v2s64}, 140 }) 141 .clampScalar(1, s32, s64) 142 .clampScalar(0, s32, s64) 143 .widenScalarToNextPow2(0) 144 .clampNumElements(0, v2s32, v4s32) 145 .clampNumElements(0, v2s64, v2s64) 146 .moreElementsToNextPow2(0) 147 .minScalarSameAs(1, 0); 148 149 getActionDefinitionsBuilder(G_PTR_ADD) 150 .legalFor({{p0, s64}, {v2p0, v2s64}}) 151 .clampScalar(1, s64, s64); 152 153 getActionDefinitionsBuilder(G_PTRMASK).legalFor({{p0, s64}}); 154 155 getActionDefinitionsBuilder({G_SDIV, G_UDIV}) 156 .legalFor({s32, s64}) 157 .libcallFor({s128}) 158 .clampScalar(0, s32, s64) 159 .widenScalarToNextPow2(0) 160 .scalarize(0); 161 162 getActionDefinitionsBuilder({G_SREM, G_UREM, G_SDIVREM, G_UDIVREM}) 163 .lowerFor({s1, s8, s16, s32, s64}); 164 165 getActionDefinitionsBuilder({G_SMULO, G_UMULO}).lowerFor({{s64, s1}}); 166 167 getActionDefinitionsBuilder({G_SMULH, G_UMULH}).legalFor({s32, s64}); 168 169 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX}) 170 .lowerIf([=](const LegalityQuery &Q) { return Q.Types[0].isScalar(); }); 171 172 getActionDefinitionsBuilder( 173 {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO}) 174 .legalFor({{s32, s1}, {s64, s1}}) 175 .clampScalar(0, s32, s64) 176 .widenScalarToNextPow2(0); 177 178 getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FNEG}) 179 .legalFor({s32, s64, v2s64, v4s32, v2s32}) 180 .clampNumElements(0, v2s32, v4s32) 181 .clampNumElements(0, v2s64, v2s64); 182 183 getActionDefinitionsBuilder(G_FREM).libcallFor({s32, s64}); 184 185 getActionDefinitionsBuilder({G_FCEIL, G_FABS, G_FSQRT, G_FFLOOR, G_FRINT, 186 G_FMA, G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND, 187 G_FNEARBYINT, G_INTRINSIC_LRINT}) 188 // If we don't have full FP16 support, then scalarize the elements of 189 // vectors containing fp16 types. 190 .fewerElementsIf( 191 [=, &ST](const LegalityQuery &Query) { 192 const auto &Ty = Query.Types[0]; 193 return Ty.isVector() && Ty.getElementType() == s16 && 194 !ST.hasFullFP16(); 195 }, 196 [=](const LegalityQuery &Query) { return std::make_pair(0, s16); }) 197 // If we don't have full FP16 support, then widen s16 to s32 if we 198 // encounter it. 199 .widenScalarIf( 200 [=, &ST](const LegalityQuery &Query) { 201 return Query.Types[0] == s16 && !ST.hasFullFP16(); 202 }, 203 [=](const LegalityQuery &Query) { return std::make_pair(0, s32); }) 204 .legalFor({s16, s32, s64, v2s32, v4s32, v2s64, v2s16, v4s16, v8s16}); 205 206 getActionDefinitionsBuilder( 207 {G_FCOS, G_FSIN, G_FLOG10, G_FLOG, G_FLOG2, G_FEXP, G_FEXP2, G_FPOW}) 208 // We need a call for these, so we always need to scalarize. 209 .scalarize(0) 210 // Regardless of FP16 support, widen 16-bit elements to 32-bits. 211 .minScalar(0, s32) 212 .libcallFor({s32, s64, v2s32, v4s32, v2s64}); 213 214 getActionDefinitionsBuilder(G_INSERT) 215 .unsupportedIf([=](const LegalityQuery &Query) { 216 return Query.Types[0].getSizeInBits() <= Query.Types[1].getSizeInBits(); 217 }) 218 .legalIf([=](const LegalityQuery &Query) { 219 const LLT &Ty0 = Query.Types[0]; 220 const LLT &Ty1 = Query.Types[1]; 221 if (Ty0 != s32 && Ty0 != s64 && Ty0 != p0) 222 return false; 223 return isPowerOf2_32(Ty1.getSizeInBits()) && 224 (Ty1.getSizeInBits() == 1 || Ty1.getSizeInBits() >= 8); 225 }) 226 .clampScalar(0, s32, s64) 227 .widenScalarToNextPow2(0) 228 .maxScalarIf(typeInSet(0, {s32}), 1, s16) 229 .maxScalarIf(typeInSet(0, {s64}), 1, s32) 230 .widenScalarToNextPow2(1); 231 232 getActionDefinitionsBuilder(G_EXTRACT) 233 .unsupportedIf([=](const LegalityQuery &Query) { 234 return Query.Types[0].getSizeInBits() >= Query.Types[1].getSizeInBits(); 235 }) 236 .legalIf([=](const LegalityQuery &Query) { 237 const LLT &Ty0 = Query.Types[0]; 238 const LLT &Ty1 = Query.Types[1]; 239 if (Ty1 != s32 && Ty1 != s64 && Ty1 != s128) 240 return false; 241 if (Ty1 == p0) 242 return true; 243 return isPowerOf2_32(Ty0.getSizeInBits()) && 244 (Ty0.getSizeInBits() == 1 || Ty0.getSizeInBits() >= 8); 245 }) 246 .clampScalar(1, s32, s128) 247 .widenScalarToNextPow2(1) 248 .maxScalarIf(typeInSet(1, {s32}), 0, s16) 249 .maxScalarIf(typeInSet(1, {s64}), 0, s32) 250 .widenScalarToNextPow2(0); 251 252 getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD}) 253 .legalForTypesWithMemDesc({{s32, p0, 8, 8}, 254 {s32, p0, 16, 8}, 255 {s32, p0, 32, 8}, 256 {s64, p0, 8, 2}, 257 {s64, p0, 16, 2}, 258 {s64, p0, 32, 4}, 259 {s64, p0, 64, 8}, 260 {p0, p0, 64, 8}, 261 {v2s32, p0, 64, 8}}) 262 .clampScalar(0, s32, s64) 263 .widenScalarToNextPow2(0) 264 // TODO: We could support sum-of-pow2's but the lowering code doesn't know 265 // how to do that yet. 266 .unsupportedIfMemSizeNotPow2() 267 // Lower anything left over into G_*EXT and G_LOAD 268 .lower(); 269 270 auto IsPtrVecPred = [=](const LegalityQuery &Query) { 271 const LLT &ValTy = Query.Types[0]; 272 if (!ValTy.isVector()) 273 return false; 274 const LLT EltTy = ValTy.getElementType(); 275 return EltTy.isPointer() && EltTy.getAddressSpace() == 0; 276 }; 277 278 getActionDefinitionsBuilder(G_LOAD) 279 .legalForTypesWithMemDesc({{s8, p0, 8, 8}, 280 {s16, p0, 16, 8}, 281 {s32, p0, 32, 8}, 282 {s64, p0, 64, 8}, 283 {p0, p0, 64, 8}, 284 {s128, p0, 128, 8}, 285 {v8s8, p0, 64, 8}, 286 {v16s8, p0, 128, 8}, 287 {v4s16, p0, 64, 8}, 288 {v8s16, p0, 128, 8}, 289 {v2s32, p0, 64, 8}, 290 {v4s32, p0, 128, 8}, 291 {v2s64, p0, 128, 8}}) 292 // These extends are also legal 293 .legalForTypesWithMemDesc({{s32, p0, 8, 8}, {s32, p0, 16, 8}}) 294 .clampScalar(0, s8, s64) 295 .lowerIfMemSizeNotPow2() 296 // Lower any any-extending loads left into G_ANYEXT and G_LOAD 297 .lowerIf([=](const LegalityQuery &Query) { 298 return Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits; 299 }) 300 .widenScalarToNextPow2(0) 301 .clampMaxNumElements(0, s8, 16) 302 .clampMaxNumElements(0, s16, 8) 303 .clampMaxNumElements(0, s32, 4) 304 .clampMaxNumElements(0, s64, 2) 305 .customIf(IsPtrVecPred); 306 307 getActionDefinitionsBuilder(G_STORE) 308 .legalForTypesWithMemDesc({{s8, p0, 8, 8}, 309 {s16, p0, 16, 8}, 310 {s32, p0, 8, 8}, 311 {s32, p0, 16, 8}, 312 {s32, p0, 32, 8}, 313 {s64, p0, 64, 8}, 314 {p0, p0, 64, 8}, 315 {s128, p0, 128, 8}, 316 {v16s8, p0, 128, 8}, 317 {v8s8, p0, 64, 8}, 318 {v4s16, p0, 64, 8}, 319 {v8s16, p0, 128, 8}, 320 {v2s32, p0, 64, 8}, 321 {v4s32, p0, 128, 8}, 322 {v2s64, p0, 128, 8}}) 323 .clampScalar(0, s8, s64) 324 .lowerIfMemSizeNotPow2() 325 .lowerIf([=](const LegalityQuery &Query) { 326 return Query.Types[0].isScalar() && 327 Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits; 328 }) 329 // Maximum: sN * k = 128 330 .clampMaxNumElements(0, s8, 16) 331 .clampMaxNumElements(0, s16, 8) 332 .clampMaxNumElements(0, s32, 4) 333 .clampMaxNumElements(0, s64, 2) 334 .customIf(IsPtrVecPred); 335 336 // Constants 337 getActionDefinitionsBuilder(G_CONSTANT) 338 .legalFor({p0, s8, s16, s32, s64}) 339 .clampScalar(0, s8, s64) 340 .widenScalarToNextPow2(0); 341 getActionDefinitionsBuilder(G_FCONSTANT) 342 .legalIf([=](const LegalityQuery &Query) { 343 const auto &Ty = Query.Types[0]; 344 if (HasFP16 && Ty == s16) 345 return true; 346 return Ty == s32 || Ty == s64 || Ty == s128; 347 }) 348 .clampScalar(0, MinFPScalar, s128); 349 350 getActionDefinitionsBuilder({G_ICMP, G_FCMP}) 351 .legalFor({{s32, s32}, 352 {s32, s64}, 353 {s32, p0}, 354 {v4s32, v4s32}, 355 {v2s32, v2s32}, 356 {v2s64, v2s64}, 357 {v2s64, v2p0}, 358 {v4s16, v4s16}, 359 {v8s16, v8s16}, 360 {v8s8, v8s8}, 361 {v16s8, v16s8}}) 362 .clampScalar(1, s32, s64) 363 .clampScalar(0, s32, s32) 364 .minScalarEltSameAsIf( 365 [=](const LegalityQuery &Query) { 366 const LLT &Ty = Query.Types[0]; 367 const LLT &SrcTy = Query.Types[1]; 368 return Ty.isVector() && !SrcTy.getElementType().isPointer() && 369 Ty.getElementType() != SrcTy.getElementType(); 370 }, 371 0, 1) 372 .minScalarOrEltIf( 373 [=](const LegalityQuery &Query) { return Query.Types[1] == v2s16; }, 374 1, s32) 375 .minScalarOrEltIf( 376 [=](const LegalityQuery &Query) { return Query.Types[1] == v2p0; }, 0, 377 s64) 378 .widenScalarOrEltToNextPow2(1) 379 .clampNumElements(0, v2s32, v4s32); 380 381 // Extensions 382 auto ExtLegalFunc = [=](const LegalityQuery &Query) { 383 unsigned DstSize = Query.Types[0].getSizeInBits(); 384 385 if (DstSize == 128 && !Query.Types[0].isVector()) 386 return false; // Extending to a scalar s128 needs narrowing. 387 388 // Make sure that we have something that will fit in a register, and 389 // make sure it's a power of 2. 390 if (DstSize < 8 || DstSize > 128 || !isPowerOf2_32(DstSize)) 391 return false; 392 393 const LLT &SrcTy = Query.Types[1]; 394 395 // Special case for s1. 396 if (SrcTy == s1) 397 return true; 398 399 // Make sure we fit in a register otherwise. Don't bother checking that 400 // the source type is below 128 bits. We shouldn't be allowing anything 401 // through which is wider than the destination in the first place. 402 unsigned SrcSize = SrcTy.getSizeInBits(); 403 if (SrcSize < 8 || !isPowerOf2_32(SrcSize)) 404 return false; 405 406 return true; 407 }; 408 getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT}) 409 .legalIf(ExtLegalFunc) 410 .clampScalar(0, s64, s64); // Just for s128, others are handled above. 411 412 getActionDefinitionsBuilder(G_TRUNC) 413 .minScalarOrEltIf( 414 [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); }, 415 0, s8) 416 .customIf([=](const LegalityQuery &Query) { 417 LLT DstTy = Query.Types[0]; 418 LLT SrcTy = Query.Types[1]; 419 return DstTy == v8s8 && SrcTy.getSizeInBits() > 128; 420 }) 421 .alwaysLegal(); 422 423 getActionDefinitionsBuilder(G_SEXT_INREG).legalFor({s32, s64}).lower(); 424 425 // FP conversions 426 getActionDefinitionsBuilder(G_FPTRUNC) 427 .legalFor( 428 {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}}) 429 .clampMaxNumElements(0, s32, 2); 430 getActionDefinitionsBuilder(G_FPEXT) 431 .legalFor( 432 {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}}) 433 .clampMaxNumElements(0, s64, 2); 434 435 // Conversions 436 getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI}) 437 .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32}) 438 .clampScalar(0, s32, s64) 439 .widenScalarToNextPow2(0) 440 .clampScalar(1, s32, s64) 441 .widenScalarToNextPow2(1); 442 443 getActionDefinitionsBuilder({G_SITOFP, G_UITOFP}) 444 .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32}) 445 .clampScalar(1, s32, s64) 446 .minScalarSameAs(1, 0) 447 .clampScalar(0, s32, s64) 448 .widenScalarToNextPow2(0); 449 450 // Control-flow 451 getActionDefinitionsBuilder(G_BRCOND).legalFor({s1, s8, s16, s32}); 452 getActionDefinitionsBuilder(G_BRINDIRECT).legalFor({p0}); 453 454 getActionDefinitionsBuilder(G_SELECT) 455 .legalFor({{s32, s1}, {s64, s1}, {p0, s1}}) 456 .clampScalar(0, s32, s64) 457 .widenScalarToNextPow2(0) 458 .minScalarEltSameAsIf(all(isVector(0), isVector(1)), 1, 0) 459 .lowerIf(isVector(0)); 460 461 // Pointer-handling 462 getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0}); 463 464 if (TM.getCodeModel() == CodeModel::Small) 465 getActionDefinitionsBuilder(G_GLOBAL_VALUE).custom(); 466 else 467 getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor({p0}); 468 469 getActionDefinitionsBuilder(G_PTRTOINT) 470 .legalForCartesianProduct({s1, s8, s16, s32, s64}, {p0}) 471 .maxScalar(0, s64) 472 .widenScalarToNextPow2(0, /*Min*/ 8); 473 474 getActionDefinitionsBuilder(G_INTTOPTR) 475 .unsupportedIf([&](const LegalityQuery &Query) { 476 return Query.Types[0].getSizeInBits() != Query.Types[1].getSizeInBits(); 477 }) 478 .legalFor({{p0, s64}}); 479 480 // Casts for 32 and 64-bit width type are just copies. 481 // Same for 128-bit width type, except they are on the FPR bank. 482 getActionDefinitionsBuilder(G_BITCAST) 483 // FIXME: This is wrong since G_BITCAST is not allowed to change the 484 // number of bits but it's what the previous code described and fixing 485 // it breaks tests. 486 .legalForCartesianProduct({s1, s8, s16, s32, s64, s128, v16s8, v8s8, v4s8, 487 v8s16, v4s16, v2s16, v4s32, v2s32, v2s64, 488 v2p0}); 489 490 getActionDefinitionsBuilder(G_VASTART).legalFor({p0}); 491 492 // va_list must be a pointer, but most sized types are pretty easy to handle 493 // as the destination. 494 getActionDefinitionsBuilder(G_VAARG) 495 .customForCartesianProduct({s8, s16, s32, s64, p0}, {p0}) 496 .clampScalar(0, s8, s64) 497 .widenScalarToNextPow2(0, /*Min*/ 8); 498 499 getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS) 500 .lowerIf( 501 all(typeInSet(0, {s8, s16, s32, s64}), typeIs(1, s1), typeIs(2, p0))); 502 503 getActionDefinitionsBuilder( 504 {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND, 505 G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, 506 G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX, G_ATOMIC_CMPXCHG}) 507 .legalIf(all( 508 typeInSet(0, {s8, s16, s32, s64}), typeIs(1, p0))); 509 510 getActionDefinitionsBuilder(G_BLOCK_ADDR).legalFor({p0}); 511 512 // Merge/Unmerge 513 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) { 514 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1; 515 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0; 516 517 auto notValidElt = [](const LegalityQuery &Query, unsigned TypeIdx) { 518 const LLT &Ty = Query.Types[TypeIdx]; 519 if (Ty.isVector()) { 520 const LLT &EltTy = Ty.getElementType(); 521 if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 64) 522 return true; 523 if (!isPowerOf2_32(EltTy.getSizeInBits())) 524 return true; 525 } 526 return false; 527 }; 528 529 // FIXME: This rule is horrible, but specifies the same as what we had 530 // before with the particularly strange definitions removed (e.g. 531 // s8 = G_MERGE_VALUES s32, s32). 532 // Part of the complexity comes from these ops being extremely flexible. For 533 // example, you can build/decompose vectors with it, concatenate vectors, 534 // etc. and in addition to this you can also bitcast with it at the same 535 // time. We've been considering breaking it up into multiple ops to make it 536 // more manageable throughout the backend. 537 getActionDefinitionsBuilder(Op) 538 // Break up vectors with weird elements into scalars 539 .fewerElementsIf( 540 [=](const LegalityQuery &Query) { return notValidElt(Query, 0); }, 541 scalarize(0)) 542 .fewerElementsIf( 543 [=](const LegalityQuery &Query) { return notValidElt(Query, 1); }, 544 scalarize(1)) 545 // Clamp the big scalar to s8-s512 and make it either a power of 2, 192, 546 // or 384. 547 .clampScalar(BigTyIdx, s8, s512) 548 .widenScalarIf( 549 [=](const LegalityQuery &Query) { 550 const LLT &Ty = Query.Types[BigTyIdx]; 551 return !isPowerOf2_32(Ty.getSizeInBits()) && 552 Ty.getSizeInBits() % 64 != 0; 553 }, 554 [=](const LegalityQuery &Query) { 555 // Pick the next power of 2, or a multiple of 64 over 128. 556 // Whichever is smaller. 557 const LLT &Ty = Query.Types[BigTyIdx]; 558 unsigned NewSizeInBits = 1 559 << Log2_32_Ceil(Ty.getSizeInBits() + 1); 560 if (NewSizeInBits >= 256) { 561 unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1); 562 if (RoundedTo < NewSizeInBits) 563 NewSizeInBits = RoundedTo; 564 } 565 return std::make_pair(BigTyIdx, LLT::scalar(NewSizeInBits)); 566 }) 567 // Clamp the little scalar to s8-s256 and make it a power of 2. It's not 568 // worth considering the multiples of 64 since 2*192 and 2*384 are not 569 // valid. 570 .clampScalar(LitTyIdx, s8, s256) 571 .widenScalarToNextPow2(LitTyIdx, /*Min*/ 8) 572 // So at this point, we have s8, s16, s32, s64, s128, s192, s256, s384, 573 // s512, <X x s8>, <X x s16>, <X x s32>, or <X x s64>. 574 // At this point it's simple enough to accept the legal types. 575 .legalIf([=](const LegalityQuery &Query) { 576 const LLT &BigTy = Query.Types[BigTyIdx]; 577 const LLT &LitTy = Query.Types[LitTyIdx]; 578 if (BigTy.isVector() && BigTy.getSizeInBits() < 32) 579 return false; 580 if (LitTy.isVector() && LitTy.getSizeInBits() < 32) 581 return false; 582 return BigTy.getSizeInBits() % LitTy.getSizeInBits() == 0; 583 }) 584 // Any vectors left are the wrong size. Scalarize them. 585 .scalarize(0) 586 .scalarize(1); 587 } 588 589 getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT) 590 .unsupportedIf([=](const LegalityQuery &Query) { 591 const LLT &EltTy = Query.Types[1].getElementType(); 592 return Query.Types[0] != EltTy; 593 }) 594 .minScalar(2, s64) 595 .legalIf([=](const LegalityQuery &Query) { 596 const LLT &VecTy = Query.Types[1]; 597 return VecTy == v2s16 || VecTy == v4s16 || VecTy == v8s16 || 598 VecTy == v4s32 || VecTy == v2s64 || VecTy == v2s32 || 599 VecTy == v16s8 || VecTy == v2s32 || VecTy == v2p0; 600 }) 601 .minScalarOrEltIf( 602 [=](const LegalityQuery &Query) { 603 // We want to promote to <M x s1> to <M x s64> if that wouldn't 604 // cause the total vec size to be > 128b. 605 return Query.Types[1].getNumElements() <= 2; 606 }, 607 0, s64) 608 .minScalarOrEltIf( 609 [=](const LegalityQuery &Query) { 610 return Query.Types[1].getNumElements() <= 4; 611 }, 612 0, s32) 613 .minScalarOrEltIf( 614 [=](const LegalityQuery &Query) { 615 return Query.Types[1].getNumElements() <= 8; 616 }, 617 0, s16) 618 .minScalarOrEltIf( 619 [=](const LegalityQuery &Query) { 620 return Query.Types[1].getNumElements() <= 16; 621 }, 622 0, s8) 623 .minScalarOrElt(0, s8); // Worst case, we need at least s8. 624 625 getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT) 626 .legalIf(typeInSet(0, {v8s16, v2s32, v4s32, v2s64})); 627 628 getActionDefinitionsBuilder(G_BUILD_VECTOR) 629 .legalFor({{v8s8, s8}, 630 {v16s8, s8}, 631 {v4s16, s16}, 632 {v8s16, s16}, 633 {v2s32, s32}, 634 {v4s32, s32}, 635 {v2p0, p0}, 636 {v2s64, s64}}) 637 .clampNumElements(0, v4s32, v4s32) 638 .clampNumElements(0, v2s64, v2s64) 639 .minScalarSameAs(1, 0); 640 641 getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC).lower(); 642 643 getActionDefinitionsBuilder(G_CTLZ) 644 .legalForCartesianProduct( 645 {s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32}) 646 .scalarize(1); 647 getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).lower(); 648 649 getActionDefinitionsBuilder(G_SHUFFLE_VECTOR) 650 .legalIf([=](const LegalityQuery &Query) { 651 const LLT &DstTy = Query.Types[0]; 652 const LLT &SrcTy = Query.Types[1]; 653 // For now just support the TBL2 variant which needs the source vectors 654 // to be the same size as the dest. 655 if (DstTy != SrcTy) 656 return false; 657 for (auto &Ty : {v2s32, v4s32, v2s64, v2p0, v16s8, v8s16}) { 658 if (DstTy == Ty) 659 return true; 660 } 661 return false; 662 }) 663 // G_SHUFFLE_VECTOR can have scalar sources (from 1 x s vectors), we 664 // just want those lowered into G_BUILD_VECTOR 665 .lowerIf([=](const LegalityQuery &Query) { 666 return !Query.Types[1].isVector(); 667 }) 668 .clampNumElements(0, v4s32, v4s32) 669 .clampNumElements(0, v2s64, v2s64); 670 671 getActionDefinitionsBuilder(G_CONCAT_VECTORS) 672 .legalFor({{v4s32, v2s32}, {v8s16, v4s16}}); 673 674 getActionDefinitionsBuilder(G_JUMP_TABLE).legalFor({{p0}, {s64}}); 675 676 getActionDefinitionsBuilder(G_BRJT).legalIf([=](const LegalityQuery &Query) { 677 return Query.Types[0] == p0 && Query.Types[1] == s64; 678 }); 679 680 getActionDefinitionsBuilder(G_DYN_STACKALLOC).lower(); 681 682 getActionDefinitionsBuilder({G_BZERO, G_MEMCPY, G_MEMMOVE, G_MEMSET}) 683 .libcall(); 684 685 // FIXME: Legal types are only legal with NEON. 686 getActionDefinitionsBuilder(G_ABS) 687 .lowerIf(isScalar(0)) 688 .legalFor(PackedVectorAllTypeList); 689 690 getActionDefinitionsBuilder(G_VECREDUCE_FADD) 691 // We only have FADDP to do reduction-like operations. Lower the rest. 692 .legalFor({{s32, v2s32}, {s64, v2s64}}) 693 .clampMaxNumElements(1, s64, 2) 694 .clampMaxNumElements(1, s32, 2) 695 .lower(); 696 697 getActionDefinitionsBuilder(G_VECREDUCE_ADD) 698 .legalFor( 699 {{s8, v16s8}, {s16, v8s16}, {s32, v4s32}, {s32, v2s32}, {s64, v2s64}}) 700 .clampMaxNumElements(1, s64, 2) 701 .clampMaxNumElements(1, s32, 4) 702 .lower(); 703 704 getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT}) 705 .lowerIf([=](const LegalityQuery &Q) { return Q.Types[0].isScalar(); }); 706 707 getActionDefinitionsBuilder({G_FSHL, G_FSHR}).lower(); 708 709 getActionDefinitionsBuilder(G_ROTR) 710 .legalFor({{s32, s64}, {s64, s64}}) 711 .customIf([=](const LegalityQuery &Q) { 712 return Q.Types[0].isScalar() && Q.Types[1].getScalarSizeInBits() < 64; 713 }) 714 .lower(); 715 getActionDefinitionsBuilder(G_ROTL).lower(); 716 717 getActionDefinitionsBuilder({G_SBFX, G_UBFX}) 718 .customFor({{s32, s32}, {s64, s64}}); 719 720 // TODO: s8, s16, s128 721 // TODO: v2s64, v2s32, v4s32, v4s16, v8s16 722 // TODO: Use generic lowering when custom lowering is not possible. 723 getActionDefinitionsBuilder(G_CTPOP) 724 .legalFor({{v8s8, v8s8}, {v16s8, v16s8}}) 725 .customFor({{s32, s32}, {s64, s64}}); 726 727 computeTables(); 728 verify(*ST.getInstrInfo()); 729 } 730 731 bool AArch64LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, 732 MachineInstr &MI) const { 733 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; 734 MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); 735 GISelChangeObserver &Observer = Helper.Observer; 736 switch (MI.getOpcode()) { 737 default: 738 // No idea what to do. 739 return false; 740 case TargetOpcode::G_VAARG: 741 return legalizeVaArg(MI, MRI, MIRBuilder); 742 case TargetOpcode::G_LOAD: 743 case TargetOpcode::G_STORE: 744 return legalizeLoadStore(MI, MRI, MIRBuilder, Observer); 745 case TargetOpcode::G_SHL: 746 case TargetOpcode::G_ASHR: 747 case TargetOpcode::G_LSHR: 748 return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer); 749 case TargetOpcode::G_GLOBAL_VALUE: 750 return legalizeSmallCMGlobalValue(MI, MRI, MIRBuilder, Observer); 751 case TargetOpcode::G_TRUNC: 752 return legalizeVectorTrunc(MI, Helper); 753 case TargetOpcode::G_SBFX: 754 case TargetOpcode::G_UBFX: 755 return legalizeBitfieldExtract(MI, MRI, Helper); 756 case TargetOpcode::G_ROTR: 757 return legalizeRotate(MI, MRI, Helper); 758 case TargetOpcode::G_CTPOP: 759 return legalizeCTPOP(MI, MRI, Helper); 760 } 761 762 llvm_unreachable("expected switch to return"); 763 } 764 765 bool AArch64LegalizerInfo::legalizeRotate(MachineInstr &MI, 766 MachineRegisterInfo &MRI, 767 LegalizerHelper &Helper) const { 768 // To allow for imported patterns to match, we ensure that the rotate amount 769 // is 64b with an extension. 770 Register AmtReg = MI.getOperand(2).getReg(); 771 LLT AmtTy = MRI.getType(AmtReg); 772 (void)AmtTy; 773 assert(AmtTy.isScalar() && "Expected a scalar rotate"); 774 assert(AmtTy.getSizeInBits() < 64 && "Expected this rotate to be legal"); 775 auto NewAmt = Helper.MIRBuilder.buildSExt(LLT::scalar(64), AmtReg); 776 Helper.Observer.changingInstr(MI); 777 MI.getOperand(2).setReg(NewAmt.getReg(0)); 778 Helper.Observer.changedInstr(MI); 779 return true; 780 } 781 782 static void extractParts(Register Reg, MachineRegisterInfo &MRI, 783 MachineIRBuilder &MIRBuilder, LLT Ty, int NumParts, 784 SmallVectorImpl<Register> &VRegs) { 785 for (int I = 0; I < NumParts; ++I) 786 VRegs.push_back(MRI.createGenericVirtualRegister(Ty)); 787 MIRBuilder.buildUnmerge(VRegs, Reg); 788 } 789 790 bool AArch64LegalizerInfo::legalizeVectorTrunc( 791 MachineInstr &MI, LegalizerHelper &Helper) const { 792 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; 793 MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); 794 // Similar to how operand splitting is done in SelectiondDAG, we can handle 795 // %res(v8s8) = G_TRUNC %in(v8s32) by generating: 796 // %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>) 797 // %lo16(<4 x s16>) = G_TRUNC %inlo 798 // %hi16(<4 x s16>) = G_TRUNC %inhi 799 // %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16 800 // %res(<8 x s8>) = G_TRUNC %in16 801 802 Register DstReg = MI.getOperand(0).getReg(); 803 Register SrcReg = MI.getOperand(1).getReg(); 804 LLT DstTy = MRI.getType(DstReg); 805 LLT SrcTy = MRI.getType(SrcReg); 806 assert(isPowerOf2_32(DstTy.getSizeInBits()) && 807 isPowerOf2_32(SrcTy.getSizeInBits())); 808 809 // Split input type. 810 LLT SplitSrcTy = SrcTy.changeNumElements(SrcTy.getNumElements() / 2); 811 // First, split the source into two smaller vectors. 812 SmallVector<Register, 2> SplitSrcs; 813 extractParts(SrcReg, MRI, MIRBuilder, SplitSrcTy, 2, SplitSrcs); 814 815 // Truncate the splits into intermediate narrower elements. 816 LLT InterTy = SplitSrcTy.changeElementSize(DstTy.getScalarSizeInBits() * 2); 817 for (unsigned I = 0; I < SplitSrcs.size(); ++I) 818 SplitSrcs[I] = MIRBuilder.buildTrunc(InterTy, SplitSrcs[I]).getReg(0); 819 820 auto Concat = MIRBuilder.buildConcatVectors( 821 DstTy.changeElementSize(DstTy.getScalarSizeInBits() * 2), SplitSrcs); 822 823 Helper.Observer.changingInstr(MI); 824 MI.getOperand(1).setReg(Concat.getReg(0)); 825 Helper.Observer.changedInstr(MI); 826 return true; 827 } 828 829 bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue( 830 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, 831 GISelChangeObserver &Observer) const { 832 assert(MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 833 // We do this custom legalization to convert G_GLOBAL_VALUE into target ADRP + 834 // G_ADD_LOW instructions. 835 // By splitting this here, we can optimize accesses in the small code model by 836 // folding in the G_ADD_LOW into the load/store offset. 837 auto &GlobalOp = MI.getOperand(1); 838 const auto* GV = GlobalOp.getGlobal(); 839 if (GV->isThreadLocal()) 840 return true; // Don't want to modify TLS vars. 841 842 auto &TM = ST->getTargetLowering()->getTargetMachine(); 843 unsigned OpFlags = ST->ClassifyGlobalReference(GV, TM); 844 845 if (OpFlags & AArch64II::MO_GOT) 846 return true; 847 848 auto Offset = GlobalOp.getOffset(); 849 Register DstReg = MI.getOperand(0).getReg(); 850 auto ADRP = MIRBuilder.buildInstr(AArch64::ADRP, {LLT::pointer(0, 64)}, {}) 851 .addGlobalAddress(GV, Offset, OpFlags | AArch64II::MO_PAGE); 852 // Set the regclass on the dest reg too. 853 MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass); 854 855 // MO_TAGGED on the page indicates a tagged address. Set the tag now. We do so 856 // by creating a MOVK that sets bits 48-63 of the register to (global address 857 // + 0x100000000 - PC) >> 48. The additional 0x100000000 offset here is to 858 // prevent an incorrect tag being generated during relocation when the the 859 // global appears before the code section. Without the offset, a global at 860 // `0x0f00'0000'0000'1000` (i.e. at `0x1000` with tag `0xf`) that's referenced 861 // by code at `0x2000` would result in `0x0f00'0000'0000'1000 - 0x2000 = 862 // 0x0eff'ffff'ffff'f000`, meaning the tag would be incorrectly set to `0xe` 863 // instead of `0xf`. 864 // This assumes that we're in the small code model so we can assume a binary 865 // size of <= 4GB, which makes the untagged PC relative offset positive. The 866 // binary must also be loaded into address range [0, 2^48). Both of these 867 // properties need to be ensured at runtime when using tagged addresses. 868 if (OpFlags & AArch64II::MO_TAGGED) { 869 assert(!Offset && 870 "Should not have folded in an offset for a tagged global!"); 871 ADRP = MIRBuilder.buildInstr(AArch64::MOVKXi, {LLT::pointer(0, 64)}, {ADRP}) 872 .addGlobalAddress(GV, 0x100000000, 873 AArch64II::MO_PREL | AArch64II::MO_G3) 874 .addImm(48); 875 MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass); 876 } 877 878 MIRBuilder.buildInstr(AArch64::G_ADD_LOW, {DstReg}, {ADRP}) 879 .addGlobalAddress(GV, Offset, 880 OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC); 881 MI.eraseFromParent(); 882 return true; 883 } 884 885 bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, 886 MachineInstr &MI) const { 887 return true; 888 } 889 890 bool AArch64LegalizerInfo::legalizeShlAshrLshr( 891 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, 892 GISelChangeObserver &Observer) const { 893 assert(MI.getOpcode() == TargetOpcode::G_ASHR || 894 MI.getOpcode() == TargetOpcode::G_LSHR || 895 MI.getOpcode() == TargetOpcode::G_SHL); 896 // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the 897 // imported patterns can select it later. Either way, it will be legal. 898 Register AmtReg = MI.getOperand(2).getReg(); 899 auto VRegAndVal = getConstantVRegValWithLookThrough(AmtReg, MRI); 900 if (!VRegAndVal) 901 return true; 902 // Check the shift amount is in range for an immediate form. 903 int64_t Amount = VRegAndVal->Value.getSExtValue(); 904 if (Amount > 31) 905 return true; // This will have to remain a register variant. 906 auto ExtCst = MIRBuilder.buildConstant(LLT::scalar(64), Amount); 907 Observer.changingInstr(MI); 908 MI.getOperand(2).setReg(ExtCst.getReg(0)); 909 Observer.changedInstr(MI); 910 return true; 911 } 912 913 bool AArch64LegalizerInfo::legalizeLoadStore( 914 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, 915 GISelChangeObserver &Observer) const { 916 assert(MI.getOpcode() == TargetOpcode::G_STORE || 917 MI.getOpcode() == TargetOpcode::G_LOAD); 918 // Here we just try to handle vector loads/stores where our value type might 919 // have pointer elements, which the SelectionDAG importer can't handle. To 920 // allow the existing patterns for s64 to fire for p0, we just try to bitcast 921 // the value to use s64 types. 922 923 // Custom legalization requires the instruction, if not deleted, must be fully 924 // legalized. In order to allow further legalization of the inst, we create 925 // a new instruction and erase the existing one. 926 927 Register ValReg = MI.getOperand(0).getReg(); 928 const LLT ValTy = MRI.getType(ValReg); 929 930 if (!ValTy.isVector() || !ValTy.getElementType().isPointer() || 931 ValTy.getElementType().getAddressSpace() != 0) { 932 LLVM_DEBUG(dbgs() << "Tried to do custom legalization on wrong load/store"); 933 return false; 934 } 935 936 unsigned PtrSize = ValTy.getElementType().getSizeInBits(); 937 const LLT NewTy = LLT::vector(ValTy.getNumElements(), PtrSize); 938 auto &MMO = **MI.memoperands_begin(); 939 if (MI.getOpcode() == TargetOpcode::G_STORE) { 940 auto Bitcast = MIRBuilder.buildBitcast(NewTy, ValReg); 941 MIRBuilder.buildStore(Bitcast.getReg(0), MI.getOperand(1), MMO); 942 } else { 943 auto NewLoad = MIRBuilder.buildLoad(NewTy, MI.getOperand(1), MMO); 944 MIRBuilder.buildBitcast(ValReg, NewLoad); 945 } 946 MI.eraseFromParent(); 947 return true; 948 } 949 950 bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI, 951 MachineRegisterInfo &MRI, 952 MachineIRBuilder &MIRBuilder) const { 953 MachineFunction &MF = MIRBuilder.getMF(); 954 Align Alignment(MI.getOperand(2).getImm()); 955 Register Dst = MI.getOperand(0).getReg(); 956 Register ListPtr = MI.getOperand(1).getReg(); 957 958 LLT PtrTy = MRI.getType(ListPtr); 959 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits()); 960 961 const unsigned PtrSize = PtrTy.getSizeInBits() / 8; 962 const Align PtrAlign = Align(PtrSize); 963 auto List = MIRBuilder.buildLoad( 964 PtrTy, ListPtr, 965 *MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOLoad, 966 PtrSize, PtrAlign)); 967 968 MachineInstrBuilder DstPtr; 969 if (Alignment > PtrAlign) { 970 // Realign the list to the actual required alignment. 971 auto AlignMinus1 = 972 MIRBuilder.buildConstant(IntPtrTy, Alignment.value() - 1); 973 auto ListTmp = MIRBuilder.buildPtrAdd(PtrTy, List, AlignMinus1.getReg(0)); 974 DstPtr = MIRBuilder.buildMaskLowPtrBits(PtrTy, ListTmp, Log2(Alignment)); 975 } else 976 DstPtr = List; 977 978 uint64_t ValSize = MRI.getType(Dst).getSizeInBits() / 8; 979 MIRBuilder.buildLoad( 980 Dst, DstPtr, 981 *MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOLoad, 982 ValSize, std::max(Alignment, PtrAlign))); 983 984 auto Size = MIRBuilder.buildConstant(IntPtrTy, alignTo(ValSize, PtrAlign)); 985 986 auto NewList = MIRBuilder.buildPtrAdd(PtrTy, DstPtr, Size.getReg(0)); 987 988 MIRBuilder.buildStore(NewList, ListPtr, 989 *MF.getMachineMemOperand(MachinePointerInfo(), 990 MachineMemOperand::MOStore, 991 PtrSize, PtrAlign)); 992 993 MI.eraseFromParent(); 994 return true; 995 } 996 997 bool AArch64LegalizerInfo::legalizeBitfieldExtract( 998 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const { 999 // Only legal if we can select immediate forms. 1000 // TODO: Lower this otherwise. 1001 return getConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI) && 1002 getConstantVRegValWithLookThrough(MI.getOperand(3).getReg(), MRI); 1003 } 1004 1005 bool AArch64LegalizerInfo::legalizeCTPOP(MachineInstr &MI, 1006 MachineRegisterInfo &MRI, 1007 LegalizerHelper &Helper) const { 1008 // While there is no integer popcount instruction, it can 1009 // be more efficiently lowered to the following sequence that uses 1010 // AdvSIMD registers/instructions as long as the copies to/from 1011 // the AdvSIMD registers are cheap. 1012 // FMOV D0, X0 // copy 64-bit int to vector, high bits zero'd 1013 // CNT V0.8B, V0.8B // 8xbyte pop-counts 1014 // ADDV B0, V0.8B // sum 8xbyte pop-counts 1015 // UMOV X0, V0.B[0] // copy byte result back to integer reg 1016 if (!ST->hasNEON() || 1017 MI.getMF()->getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) 1018 return false; 1019 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; 1020 Register Dst = MI.getOperand(0).getReg(); 1021 Register Val = MI.getOperand(1).getReg(); 1022 LLT Ty = MRI.getType(Val); 1023 1024 // TODO: Handle vector types. 1025 assert(!Ty.isVector() && "Vector types not handled yet!"); 1026 assert(Ty == MRI.getType(Dst) && 1027 "Expected src and dst to have the same type!"); 1028 // TODO: Handle s128. 1029 unsigned Size = Ty.getSizeInBits(); 1030 assert((Size == 32 || Size == 64) && "Expected only 32 or 64 bit scalars!"); 1031 if (Size == 32) 1032 Val = MIRBuilder.buildZExt(LLT::scalar(64), Val).getReg(0); 1033 const LLT V8S8 = LLT::vector(8, LLT::scalar(8)); 1034 Val = MIRBuilder.buildBitcast(V8S8, Val).getReg(0); 1035 auto CTPOP = MIRBuilder.buildCTPOP(V8S8, Val); 1036 auto UADDLV = 1037 MIRBuilder 1038 .buildIntrinsic(Intrinsic::aarch64_neon_uaddlv, {LLT::scalar(32)}, 1039 /*HasSideEffects = */ false) 1040 .addUse(CTPOP.getReg(0)); 1041 if (Size == 64) 1042 MIRBuilder.buildZExt(Dst, UADDLV); 1043 else 1044 UADDLV->getOperand(0).setReg(Dst); 1045 MI.eraseFromParent(); 1046 return true; 1047 } 1048