1 //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file defines an instruction selector for the ARM target. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #define DEBUG_TYPE "arm-isel" 15 #include "ARM.h" 16 #include "ARMBaseInstrInfo.h" 17 #include "ARMTargetMachine.h" 18 #include "MCTargetDesc/ARMAddressingModes.h" 19 #include "llvm/CallingConv.h" 20 #include "llvm/Constants.h" 21 #include "llvm/DerivedTypes.h" 22 #include "llvm/Function.h" 23 #include "llvm/Intrinsics.h" 24 #include "llvm/LLVMContext.h" 25 #include "llvm/CodeGen/MachineFrameInfo.h" 26 #include "llvm/CodeGen/MachineFunction.h" 27 #include "llvm/CodeGen/MachineInstrBuilder.h" 28 #include "llvm/CodeGen/SelectionDAG.h" 29 #include "llvm/CodeGen/SelectionDAGISel.h" 30 #include "llvm/Target/TargetLowering.h" 31 #include "llvm/Target/TargetOptions.h" 32 #include "llvm/Support/CommandLine.h" 33 #include "llvm/Support/Compiler.h" 34 #include "llvm/Support/Debug.h" 35 #include "llvm/Support/ErrorHandling.h" 36 #include "llvm/Support/raw_ostream.h" 37 38 using namespace llvm; 39 40 static cl::opt<bool> 41 DisableShifterOp("disable-shifter-op", cl::Hidden, 42 cl::desc("Disable isel of shifter-op"), 43 cl::init(false)); 44 45 static cl::opt<bool> 46 CheckVMLxHazard("check-vmlx-hazard", cl::Hidden, 47 cl::desc("Check fp vmla / vmls hazard at isel time"), 48 cl::init(true)); 49 50 //===--------------------------------------------------------------------===// 51 /// ARMDAGToDAGISel - ARM specific code to select ARM machine 52 /// instructions for SelectionDAG operations. 53 /// 54 namespace { 55 56 enum AddrMode2Type { 57 AM2_BASE, // Simple AM2 (+-imm12) 58 AM2_SHOP // Shifter-op AM2 59 }; 60 61 class ARMDAGToDAGISel : public SelectionDAGISel { 62 ARMBaseTargetMachine &TM; 63 const ARMBaseInstrInfo *TII; 64 65 /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can 66 /// make the right decision when generating code for different targets. 67 const ARMSubtarget *Subtarget; 68 69 public: 70 explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, 71 CodeGenOpt::Level OptLevel) 72 : SelectionDAGISel(tm, OptLevel), TM(tm), 73 TII(static_cast<const ARMBaseInstrInfo*>(TM.getInstrInfo())), 74 Subtarget(&TM.getSubtarget<ARMSubtarget>()) { 75 } 76 77 virtual const char *getPassName() const { 78 return "ARM Instruction Selection"; 79 } 80 81 /// getI32Imm - Return a target constant of type i32 with the specified 82 /// value. 83 inline SDValue getI32Imm(unsigned Imm) { 84 return CurDAG->getTargetConstant(Imm, MVT::i32); 85 } 86 87 SDNode *Select(SDNode *N); 88 89 90 bool hasNoVMLxHazardUse(SDNode *N) const; 91 bool isShifterOpProfitable(const SDValue &Shift, 92 ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt); 93 bool SelectRegShifterOperand(SDValue N, SDValue &A, 94 SDValue &B, SDValue &C, 95 bool CheckProfitability = true); 96 bool SelectImmShifterOperand(SDValue N, SDValue &A, 97 SDValue &B, bool CheckProfitability = true); 98 bool SelectShiftRegShifterOperand(SDValue N, SDValue &A, 99 SDValue &B, SDValue &C) { 100 // Don't apply the profitability check 101 return SelectRegShifterOperand(N, A, B, C, false); 102 } 103 bool SelectShiftImmShifterOperand(SDValue N, SDValue &A, 104 SDValue &B) { 105 // Don't apply the profitability check 106 return SelectImmShifterOperand(N, A, B, false); 107 } 108 109 bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm); 110 bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc); 111 112 AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base, 113 SDValue &Offset, SDValue &Opc); 114 bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset, 115 SDValue &Opc) { 116 return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE; 117 } 118 119 bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset, 120 SDValue &Opc) { 121 return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP; 122 } 123 124 bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset, 125 SDValue &Opc) { 126 SelectAddrMode2Worker(N, Base, Offset, Opc); 127 // return SelectAddrMode2ShOp(N, Base, Offset, Opc); 128 // This always matches one way or another. 129 return true; 130 } 131 132 bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, 133 SDValue &Offset, SDValue &Opc); 134 bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, 135 SDValue &Offset, SDValue &Opc); 136 bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, 137 SDValue &Offset, SDValue &Opc); 138 bool SelectAddrOffsetNone(SDValue N, SDValue &Base); 139 bool SelectAddrMode3(SDValue N, SDValue &Base, 140 SDValue &Offset, SDValue &Opc); 141 bool SelectAddrMode3Offset(SDNode *Op, SDValue N, 142 SDValue &Offset, SDValue &Opc); 143 bool SelectAddrMode5(SDValue N, SDValue &Base, 144 SDValue &Offset); 145 bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align); 146 bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset); 147 148 bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label); 149 150 // Thumb Addressing Modes: 151 bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset); 152 bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset, 153 unsigned Scale); 154 bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset); 155 bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset); 156 bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset); 157 bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base, 158 SDValue &OffImm); 159 bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, 160 SDValue &OffImm); 161 bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, 162 SDValue &OffImm); 163 bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, 164 SDValue &OffImm); 165 bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm); 166 167 // Thumb 2 Addressing Modes: 168 bool SelectT2ShifterOperandReg(SDValue N, 169 SDValue &BaseReg, SDValue &Opc); 170 bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm); 171 bool SelectT2AddrModeImm8(SDValue N, SDValue &Base, 172 SDValue &OffImm); 173 bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, 174 SDValue &OffImm); 175 bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base, 176 SDValue &OffReg, SDValue &ShImm); 177 178 inline bool is_so_imm(unsigned Imm) const { 179 return ARM_AM::getSOImmVal(Imm) != -1; 180 } 181 182 inline bool is_so_imm_not(unsigned Imm) const { 183 return ARM_AM::getSOImmVal(~Imm) != -1; 184 } 185 186 inline bool is_t2_so_imm(unsigned Imm) const { 187 return ARM_AM::getT2SOImmVal(Imm) != -1; 188 } 189 190 inline bool is_t2_so_imm_not(unsigned Imm) const { 191 return ARM_AM::getT2SOImmVal(~Imm) != -1; 192 } 193 194 // Include the pieces autogenerated from the target description. 195 #include "ARMGenDAGISel.inc" 196 197 private: 198 /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for 199 /// ARM. 200 SDNode *SelectARMIndexedLoad(SDNode *N); 201 SDNode *SelectT2IndexedLoad(SDNode *N); 202 203 /// SelectVLD - Select NEON load intrinsics. NumVecs should be 204 /// 1, 2, 3 or 4. The opcode arrays specify the instructions used for 205 /// loads of D registers and even subregs and odd subregs of Q registers. 206 /// For NumVecs <= 2, QOpcodes1 is not used. 207 SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, 208 const uint16_t *DOpcodes, 209 const uint16_t *QOpcodes0, const uint16_t *QOpcodes1); 210 211 /// SelectVST - Select NEON store intrinsics. NumVecs should 212 /// be 1, 2, 3 or 4. The opcode arrays specify the instructions used for 213 /// stores of D registers and even subregs and odd subregs of Q registers. 214 /// For NumVecs <= 2, QOpcodes1 is not used. 215 SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, 216 const uint16_t *DOpcodes, 217 const uint16_t *QOpcodes0, const uint16_t *QOpcodes1); 218 219 /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should 220 /// be 2, 3 or 4. The opcode arrays specify the instructions used for 221 /// load/store of D registers and Q registers. 222 SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, 223 bool isUpdating, unsigned NumVecs, 224 const uint16_t *DOpcodes, const uint16_t *QOpcodes); 225 226 /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs 227 /// should be 2, 3 or 4. The opcode array specifies the instructions used 228 /// for loading D registers. (Q registers are not supported.) 229 SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs, 230 const uint16_t *Opcodes); 231 232 /// SelectVTBL - Select NEON VTBL and VTBX intrinsics. NumVecs should be 2, 233 /// 3 or 4. These are custom-selected so that a REG_SEQUENCE can be 234 /// generated to force the table registers to be consecutive. 235 SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc); 236 237 /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM. 238 SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned); 239 240 /// SelectCMOVOp - Select CMOV instructions for ARM. 241 SDNode *SelectCMOVOp(SDNode *N); 242 SDNode *SelectConditionalOp(SDNode *N); 243 SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 244 ARMCC::CondCodes CCVal, SDValue CCR, 245 SDValue InFlag); 246 SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 247 ARMCC::CondCodes CCVal, SDValue CCR, 248 SDValue InFlag); 249 SDNode *SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 250 ARMCC::CondCodes CCVal, SDValue CCR, 251 SDValue InFlag); 252 SDNode *SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 253 ARMCC::CondCodes CCVal, SDValue CCR, 254 SDValue InFlag); 255 256 // Select special operations if node forms integer ABS pattern 257 SDNode *SelectABSOp(SDNode *N); 258 259 SDNode *SelectConcatVector(SDNode *N); 260 261 SDNode *SelectAtomic64(SDNode *Node, unsigned Opc); 262 263 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 264 /// inline asm expressions. 265 virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, 266 char ConstraintCode, 267 std::vector<SDValue> &OutOps); 268 269 // Form pairs of consecutive S, D, or Q registers. 270 SDNode *PairSRegs(EVT VT, SDValue V0, SDValue V1); 271 SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1); 272 SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1); 273 274 // Form sequences of 4 consecutive S, D, or Q registers. 275 SDNode *QuadSRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 276 SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 277 SDNode *QuadQRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 278 279 // Get the alignment operand for a NEON VLD or VST instruction. 280 SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector); 281 }; 282 } 283 284 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant 285 /// operand. If so Imm will receive the 32-bit value. 286 static bool isInt32Immediate(SDNode *N, unsigned &Imm) { 287 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { 288 Imm = cast<ConstantSDNode>(N)->getZExtValue(); 289 return true; 290 } 291 return false; 292 } 293 294 // isInt32Immediate - This method tests to see if a constant operand. 295 // If so Imm will receive the 32 bit value. 296 static bool isInt32Immediate(SDValue N, unsigned &Imm) { 297 return isInt32Immediate(N.getNode(), Imm); 298 } 299 300 // isOpcWithIntImmediate - This method tests to see if the node is a specific 301 // opcode and that it has a immediate integer right operand. 302 // If so Imm will receive the 32 bit value. 303 static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { 304 return N->getOpcode() == Opc && 305 isInt32Immediate(N->getOperand(1).getNode(), Imm); 306 } 307 308 /// \brief Check whether a particular node is a constant value representable as 309 /// (N * Scale) where (N in [\arg RangeMin, \arg RangeMax). 310 /// 311 /// \param ScaledConstant [out] - On success, the pre-scaled constant value. 312 static bool isScaledConstantInRange(SDValue Node, int Scale, 313 int RangeMin, int RangeMax, 314 int &ScaledConstant) { 315 assert(Scale > 0 && "Invalid scale!"); 316 317 // Check that this is a constant. 318 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node); 319 if (!C) 320 return false; 321 322 ScaledConstant = (int) C->getZExtValue(); 323 if ((ScaledConstant % Scale) != 0) 324 return false; 325 326 ScaledConstant /= Scale; 327 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax; 328 } 329 330 /// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS 331 /// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at 332 /// least on current ARM implementations) which should be avoidded. 333 bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const { 334 if (OptLevel == CodeGenOpt::None) 335 return true; 336 337 if (!CheckVMLxHazard) 338 return true; 339 340 if (!Subtarget->isCortexA8() && !Subtarget->isCortexA9()) 341 return true; 342 343 if (!N->hasOneUse()) 344 return false; 345 346 SDNode *Use = *N->use_begin(); 347 if (Use->getOpcode() == ISD::CopyToReg) 348 return true; 349 if (Use->isMachineOpcode()) { 350 const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode()); 351 if (MCID.mayStore()) 352 return true; 353 unsigned Opcode = MCID.getOpcode(); 354 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD) 355 return true; 356 // vmlx feeding into another vmlx. We actually want to unfold 357 // the use later in the MLxExpansion pass. e.g. 358 // vmla 359 // vmla (stall 8 cycles) 360 // 361 // vmul (5 cycles) 362 // vadd (5 cycles) 363 // vmla 364 // This adds up to about 18 - 19 cycles. 365 // 366 // vmla 367 // vmul (stall 4 cycles) 368 // vadd adds up to about 14 cycles. 369 return TII->isFpMLxInstruction(Opcode); 370 } 371 372 return false; 373 } 374 375 bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift, 376 ARM_AM::ShiftOpc ShOpcVal, 377 unsigned ShAmt) { 378 if (!Subtarget->isCortexA9()) 379 return true; 380 if (Shift.hasOneUse()) 381 return true; 382 // R << 2 is free. 383 return ShOpcVal == ARM_AM::lsl && ShAmt == 2; 384 } 385 386 bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N, 387 SDValue &BaseReg, 388 SDValue &Opc, 389 bool CheckProfitability) { 390 if (DisableShifterOp) 391 return false; 392 393 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 394 395 // Don't match base register only case. That is matched to a separate 396 // lower complexity pattern with explicit register operand. 397 if (ShOpcVal == ARM_AM::no_shift) return false; 398 399 BaseReg = N.getOperand(0); 400 unsigned ShImmVal = 0; 401 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 402 if (!RHS) return false; 403 ShImmVal = RHS->getZExtValue() & 31; 404 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), 405 MVT::i32); 406 return true; 407 } 408 409 bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N, 410 SDValue &BaseReg, 411 SDValue &ShReg, 412 SDValue &Opc, 413 bool CheckProfitability) { 414 if (DisableShifterOp) 415 return false; 416 417 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 418 419 // Don't match base register only case. That is matched to a separate 420 // lower complexity pattern with explicit register operand. 421 if (ShOpcVal == ARM_AM::no_shift) return false; 422 423 BaseReg = N.getOperand(0); 424 unsigned ShImmVal = 0; 425 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 426 if (RHS) return false; 427 428 ShReg = N.getOperand(1); 429 if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal)) 430 return false; 431 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), 432 MVT::i32); 433 return true; 434 } 435 436 437 bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N, 438 SDValue &Base, 439 SDValue &OffImm) { 440 // Match simple R + imm12 operands. 441 442 // Base only. 443 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 444 !CurDAG->isBaseWithConstantOffset(N)) { 445 if (N.getOpcode() == ISD::FrameIndex) { 446 // Match frame index. 447 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 448 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 449 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 450 return true; 451 } 452 453 if (N.getOpcode() == ARMISD::Wrapper && 454 !(Subtarget->useMovt() && 455 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { 456 Base = N.getOperand(0); 457 } else 458 Base = N; 459 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 460 return true; 461 } 462 463 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 464 int RHSC = (int)RHS->getZExtValue(); 465 if (N.getOpcode() == ISD::SUB) 466 RHSC = -RHSC; 467 468 if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned) 469 Base = N.getOperand(0); 470 if (Base.getOpcode() == ISD::FrameIndex) { 471 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 472 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 473 } 474 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 475 return true; 476 } 477 } 478 479 // Base only. 480 Base = N; 481 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 482 return true; 483 } 484 485 486 487 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, 488 SDValue &Opc) { 489 if (N.getOpcode() == ISD::MUL && 490 (!Subtarget->isCortexA9() || N.hasOneUse())) { 491 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 492 // X * [3,5,9] -> X + X * [2,4,8] etc. 493 int RHSC = (int)RHS->getZExtValue(); 494 if (RHSC & 1) { 495 RHSC = RHSC & ~1; 496 ARM_AM::AddrOpc AddSub = ARM_AM::add; 497 if (RHSC < 0) { 498 AddSub = ARM_AM::sub; 499 RHSC = - RHSC; 500 } 501 if (isPowerOf2_32(RHSC)) { 502 unsigned ShAmt = Log2_32(RHSC); 503 Base = Offset = N.getOperand(0); 504 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, 505 ARM_AM::lsl), 506 MVT::i32); 507 return true; 508 } 509 } 510 } 511 } 512 513 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 514 // ISD::OR that is equivalent to an ISD::ADD. 515 !CurDAG->isBaseWithConstantOffset(N)) 516 return false; 517 518 // Leave simple R +/- imm12 operands for LDRi12 519 if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) { 520 int RHSC; 521 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 522 -0x1000+1, 0x1000, RHSC)) // 12 bits. 523 return false; 524 } 525 526 // Otherwise this is R +/- [possibly shifted] R. 527 ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add; 528 ARM_AM::ShiftOpc ShOpcVal = 529 ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); 530 unsigned ShAmt = 0; 531 532 Base = N.getOperand(0); 533 Offset = N.getOperand(1); 534 535 if (ShOpcVal != ARM_AM::no_shift) { 536 // Check to see if the RHS of the shift is a constant, if not, we can't fold 537 // it. 538 if (ConstantSDNode *Sh = 539 dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { 540 ShAmt = Sh->getZExtValue(); 541 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt)) 542 Offset = N.getOperand(1).getOperand(0); 543 else { 544 ShAmt = 0; 545 ShOpcVal = ARM_AM::no_shift; 546 } 547 } else { 548 ShOpcVal = ARM_AM::no_shift; 549 } 550 } 551 552 // Try matching (R shl C) + (R). 553 if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && 554 !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) { 555 ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); 556 if (ShOpcVal != ARM_AM::no_shift) { 557 // Check to see if the RHS of the shift is a constant, if not, we can't 558 // fold it. 559 if (ConstantSDNode *Sh = 560 dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { 561 ShAmt = Sh->getZExtValue(); 562 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { 563 Offset = N.getOperand(0).getOperand(0); 564 Base = N.getOperand(1); 565 } else { 566 ShAmt = 0; 567 ShOpcVal = ARM_AM::no_shift; 568 } 569 } else { 570 ShOpcVal = ARM_AM::no_shift; 571 } 572 } 573 } 574 575 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 576 MVT::i32); 577 return true; 578 } 579 580 581 //----- 582 583 AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N, 584 SDValue &Base, 585 SDValue &Offset, 586 SDValue &Opc) { 587 if (N.getOpcode() == ISD::MUL && 588 (!Subtarget->isCortexA9() || N.hasOneUse())) { 589 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 590 // X * [3,5,9] -> X + X * [2,4,8] etc. 591 int RHSC = (int)RHS->getZExtValue(); 592 if (RHSC & 1) { 593 RHSC = RHSC & ~1; 594 ARM_AM::AddrOpc AddSub = ARM_AM::add; 595 if (RHSC < 0) { 596 AddSub = ARM_AM::sub; 597 RHSC = - RHSC; 598 } 599 if (isPowerOf2_32(RHSC)) { 600 unsigned ShAmt = Log2_32(RHSC); 601 Base = Offset = N.getOperand(0); 602 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, 603 ARM_AM::lsl), 604 MVT::i32); 605 return AM2_SHOP; 606 } 607 } 608 } 609 } 610 611 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 612 // ISD::OR that is equivalent to an ADD. 613 !CurDAG->isBaseWithConstantOffset(N)) { 614 Base = N; 615 if (N.getOpcode() == ISD::FrameIndex) { 616 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 617 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 618 } else if (N.getOpcode() == ARMISD::Wrapper && 619 !(Subtarget->useMovt() && 620 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { 621 Base = N.getOperand(0); 622 } 623 Offset = CurDAG->getRegister(0, MVT::i32); 624 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, 625 ARM_AM::no_shift), 626 MVT::i32); 627 return AM2_BASE; 628 } 629 630 // Match simple R +/- imm12 operands. 631 if (N.getOpcode() != ISD::SUB) { 632 int RHSC; 633 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 634 -0x1000+1, 0x1000, RHSC)) { // 12 bits. 635 Base = N.getOperand(0); 636 if (Base.getOpcode() == ISD::FrameIndex) { 637 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 638 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 639 } 640 Offset = CurDAG->getRegister(0, MVT::i32); 641 642 ARM_AM::AddrOpc AddSub = ARM_AM::add; 643 if (RHSC < 0) { 644 AddSub = ARM_AM::sub; 645 RHSC = - RHSC; 646 } 647 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC, 648 ARM_AM::no_shift), 649 MVT::i32); 650 return AM2_BASE; 651 } 652 } 653 654 if (Subtarget->isCortexA9() && !N.hasOneUse()) { 655 // Compute R +/- (R << N) and reuse it. 656 Base = N; 657 Offset = CurDAG->getRegister(0, MVT::i32); 658 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, 659 ARM_AM::no_shift), 660 MVT::i32); 661 return AM2_BASE; 662 } 663 664 // Otherwise this is R +/- [possibly shifted] R. 665 ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub; 666 ARM_AM::ShiftOpc ShOpcVal = 667 ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); 668 unsigned ShAmt = 0; 669 670 Base = N.getOperand(0); 671 Offset = N.getOperand(1); 672 673 if (ShOpcVal != ARM_AM::no_shift) { 674 // Check to see if the RHS of the shift is a constant, if not, we can't fold 675 // it. 676 if (ConstantSDNode *Sh = 677 dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { 678 ShAmt = Sh->getZExtValue(); 679 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt)) 680 Offset = N.getOperand(1).getOperand(0); 681 else { 682 ShAmt = 0; 683 ShOpcVal = ARM_AM::no_shift; 684 } 685 } else { 686 ShOpcVal = ARM_AM::no_shift; 687 } 688 } 689 690 // Try matching (R shl C) + (R). 691 if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && 692 !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) { 693 ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); 694 if (ShOpcVal != ARM_AM::no_shift) { 695 // Check to see if the RHS of the shift is a constant, if not, we can't 696 // fold it. 697 if (ConstantSDNode *Sh = 698 dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { 699 ShAmt = Sh->getZExtValue(); 700 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { 701 Offset = N.getOperand(0).getOperand(0); 702 Base = N.getOperand(1); 703 } else { 704 ShAmt = 0; 705 ShOpcVal = ARM_AM::no_shift; 706 } 707 } else { 708 ShOpcVal = ARM_AM::no_shift; 709 } 710 } 711 } 712 713 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 714 MVT::i32); 715 return AM2_SHOP; 716 } 717 718 bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, 719 SDValue &Offset, SDValue &Opc) { 720 unsigned Opcode = Op->getOpcode(); 721 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 722 ? cast<LoadSDNode>(Op)->getAddressingMode() 723 : cast<StoreSDNode>(Op)->getAddressingMode(); 724 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 725 ? ARM_AM::add : ARM_AM::sub; 726 int Val; 727 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) 728 return false; 729 730 Offset = N; 731 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 732 unsigned ShAmt = 0; 733 if (ShOpcVal != ARM_AM::no_shift) { 734 // Check to see if the RHS of the shift is a constant, if not, we can't fold 735 // it. 736 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 737 ShAmt = Sh->getZExtValue(); 738 if (isShifterOpProfitable(N, ShOpcVal, ShAmt)) 739 Offset = N.getOperand(0); 740 else { 741 ShAmt = 0; 742 ShOpcVal = ARM_AM::no_shift; 743 } 744 } else { 745 ShOpcVal = ARM_AM::no_shift; 746 } 747 } 748 749 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 750 MVT::i32); 751 return true; 752 } 753 754 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, 755 SDValue &Offset, SDValue &Opc) { 756 unsigned Opcode = Op->getOpcode(); 757 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 758 ? cast<LoadSDNode>(Op)->getAddressingMode() 759 : cast<StoreSDNode>(Op)->getAddressingMode(); 760 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 761 ? ARM_AM::add : ARM_AM::sub; 762 int Val; 763 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. 764 if (AddSub == ARM_AM::sub) Val *= -1; 765 Offset = CurDAG->getRegister(0, MVT::i32); 766 Opc = CurDAG->getTargetConstant(Val, MVT::i32); 767 return true; 768 } 769 770 return false; 771 } 772 773 774 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, 775 SDValue &Offset, SDValue &Opc) { 776 unsigned Opcode = Op->getOpcode(); 777 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 778 ? cast<LoadSDNode>(Op)->getAddressingMode() 779 : cast<StoreSDNode>(Op)->getAddressingMode(); 780 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 781 ? ARM_AM::add : ARM_AM::sub; 782 int Val; 783 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. 784 Offset = CurDAG->getRegister(0, MVT::i32); 785 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val, 786 ARM_AM::no_shift), 787 MVT::i32); 788 return true; 789 } 790 791 return false; 792 } 793 794 bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) { 795 Base = N; 796 return true; 797 } 798 799 bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N, 800 SDValue &Base, SDValue &Offset, 801 SDValue &Opc) { 802 if (N.getOpcode() == ISD::SUB) { 803 // X - C is canonicalize to X + -C, no need to handle it here. 804 Base = N.getOperand(0); 805 Offset = N.getOperand(1); 806 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32); 807 return true; 808 } 809 810 if (!CurDAG->isBaseWithConstantOffset(N)) { 811 Base = N; 812 if (N.getOpcode() == ISD::FrameIndex) { 813 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 814 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 815 } 816 Offset = CurDAG->getRegister(0, MVT::i32); 817 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32); 818 return true; 819 } 820 821 // If the RHS is +/- imm8, fold into addr mode. 822 int RHSC; 823 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 824 -256 + 1, 256, RHSC)) { // 8 bits. 825 Base = N.getOperand(0); 826 if (Base.getOpcode() == ISD::FrameIndex) { 827 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 828 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 829 } 830 Offset = CurDAG->getRegister(0, MVT::i32); 831 832 ARM_AM::AddrOpc AddSub = ARM_AM::add; 833 if (RHSC < 0) { 834 AddSub = ARM_AM::sub; 835 RHSC = -RHSC; 836 } 837 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32); 838 return true; 839 } 840 841 Base = N.getOperand(0); 842 Offset = N.getOperand(1); 843 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32); 844 return true; 845 } 846 847 bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N, 848 SDValue &Offset, SDValue &Opc) { 849 unsigned Opcode = Op->getOpcode(); 850 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 851 ? cast<LoadSDNode>(Op)->getAddressingMode() 852 : cast<StoreSDNode>(Op)->getAddressingMode(); 853 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 854 ? ARM_AM::add : ARM_AM::sub; 855 int Val; 856 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits. 857 Offset = CurDAG->getRegister(0, MVT::i32); 858 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32); 859 return true; 860 } 861 862 Offset = N; 863 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32); 864 return true; 865 } 866 867 bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N, 868 SDValue &Base, SDValue &Offset) { 869 if (!CurDAG->isBaseWithConstantOffset(N)) { 870 Base = N; 871 if (N.getOpcode() == ISD::FrameIndex) { 872 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 873 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 874 } else if (N.getOpcode() == ARMISD::Wrapper && 875 !(Subtarget->useMovt() && 876 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { 877 Base = N.getOperand(0); 878 } 879 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), 880 MVT::i32); 881 return true; 882 } 883 884 // If the RHS is +/- imm8, fold into addr mode. 885 int RHSC; 886 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 887 -256 + 1, 256, RHSC)) { 888 Base = N.getOperand(0); 889 if (Base.getOpcode() == ISD::FrameIndex) { 890 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 891 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 892 } 893 894 ARM_AM::AddrOpc AddSub = ARM_AM::add; 895 if (RHSC < 0) { 896 AddSub = ARM_AM::sub; 897 RHSC = -RHSC; 898 } 899 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC), 900 MVT::i32); 901 return true; 902 } 903 904 Base = N; 905 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), 906 MVT::i32); 907 return true; 908 } 909 910 bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr, 911 SDValue &Align) { 912 Addr = N; 913 914 unsigned Alignment = 0; 915 if (LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Parent)) { 916 // This case occurs only for VLD1-lane/dup and VST1-lane instructions. 917 // The maximum alignment is equal to the memory size being referenced. 918 unsigned LSNAlign = LSN->getAlignment(); 919 unsigned MemSize = LSN->getMemoryVT().getSizeInBits() / 8; 920 if (LSNAlign >= MemSize && MemSize > 1) 921 Alignment = MemSize; 922 } else { 923 // All other uses of addrmode6 are for intrinsics. For now just record 924 // the raw alignment value; it will be refined later based on the legal 925 // alignment operands for the intrinsic. 926 Alignment = cast<MemIntrinsicSDNode>(Parent)->getAlignment(); 927 } 928 929 Align = CurDAG->getTargetConstant(Alignment, MVT::i32); 930 return true; 931 } 932 933 bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N, 934 SDValue &Offset) { 935 LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op); 936 ISD::MemIndexedMode AM = LdSt->getAddressingMode(); 937 if (AM != ISD::POST_INC) 938 return false; 939 Offset = N; 940 if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) { 941 if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits()) 942 Offset = CurDAG->getRegister(0, MVT::i32); 943 } 944 return true; 945 } 946 947 bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N, 948 SDValue &Offset, SDValue &Label) { 949 if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) { 950 Offset = N.getOperand(0); 951 SDValue N1 = N.getOperand(1); 952 Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(), 953 MVT::i32); 954 return true; 955 } 956 957 return false; 958 } 959 960 961 //===----------------------------------------------------------------------===// 962 // Thumb Addressing Modes 963 //===----------------------------------------------------------------------===// 964 965 bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, 966 SDValue &Base, SDValue &Offset){ 967 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) { 968 ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N); 969 if (!NC || !NC->isNullValue()) 970 return false; 971 972 Base = Offset = N; 973 return true; 974 } 975 976 Base = N.getOperand(0); 977 Offset = N.getOperand(1); 978 return true; 979 } 980 981 bool 982 ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base, 983 SDValue &Offset, unsigned Scale) { 984 if (Scale == 4) { 985 SDValue TmpBase, TmpOffImm; 986 if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm)) 987 return false; // We want to select tLDRspi / tSTRspi instead. 988 989 if (N.getOpcode() == ARMISD::Wrapper && 990 N.getOperand(0).getOpcode() == ISD::TargetConstantPool) 991 return false; // We want to select tLDRpci instead. 992 } 993 994 if (!CurDAG->isBaseWithConstantOffset(N)) 995 return false; 996 997 // Thumb does not have [sp, r] address mode. 998 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 999 RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1)); 1000 if ((LHSR && LHSR->getReg() == ARM::SP) || 1001 (RHSR && RHSR->getReg() == ARM::SP)) 1002 return false; 1003 1004 // FIXME: Why do we explicitly check for a match here and then return false? 1005 // Presumably to allow something else to match, but shouldn't this be 1006 // documented? 1007 int RHSC; 1008 if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) 1009 return false; 1010 1011 Base = N.getOperand(0); 1012 Offset = N.getOperand(1); 1013 return true; 1014 } 1015 1016 bool 1017 ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N, 1018 SDValue &Base, 1019 SDValue &Offset) { 1020 return SelectThumbAddrModeRI(N, Base, Offset, 1); 1021 } 1022 1023 bool 1024 ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N, 1025 SDValue &Base, 1026 SDValue &Offset) { 1027 return SelectThumbAddrModeRI(N, Base, Offset, 2); 1028 } 1029 1030 bool 1031 ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N, 1032 SDValue &Base, 1033 SDValue &Offset) { 1034 return SelectThumbAddrModeRI(N, Base, Offset, 4); 1035 } 1036 1037 bool 1038 ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, 1039 SDValue &Base, SDValue &OffImm) { 1040 if (Scale == 4) { 1041 SDValue TmpBase, TmpOffImm; 1042 if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm)) 1043 return false; // We want to select tLDRspi / tSTRspi instead. 1044 1045 if (N.getOpcode() == ARMISD::Wrapper && 1046 N.getOperand(0).getOpcode() == ISD::TargetConstantPool) 1047 return false; // We want to select tLDRpci instead. 1048 } 1049 1050 if (!CurDAG->isBaseWithConstantOffset(N)) { 1051 if (N.getOpcode() == ARMISD::Wrapper && 1052 !(Subtarget->useMovt() && 1053 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { 1054 Base = N.getOperand(0); 1055 } else { 1056 Base = N; 1057 } 1058 1059 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1060 return true; 1061 } 1062 1063 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 1064 RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1)); 1065 if ((LHSR && LHSR->getReg() == ARM::SP) || 1066 (RHSR && RHSR->getReg() == ARM::SP)) { 1067 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0)); 1068 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1069 unsigned LHSC = LHS ? LHS->getZExtValue() : 0; 1070 unsigned RHSC = RHS ? RHS->getZExtValue() : 0; 1071 1072 // Thumb does not have [sp, #imm5] address mode for non-zero imm5. 1073 if (LHSC != 0 || RHSC != 0) return false; 1074 1075 Base = N; 1076 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1077 return true; 1078 } 1079 1080 // If the RHS is + imm5 * scale, fold into addr mode. 1081 int RHSC; 1082 if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) { 1083 Base = N.getOperand(0); 1084 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1085 return true; 1086 } 1087 1088 Base = N.getOperand(0); 1089 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1090 return true; 1091 } 1092 1093 bool 1094 ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, 1095 SDValue &OffImm) { 1096 return SelectThumbAddrModeImm5S(N, 4, Base, OffImm); 1097 } 1098 1099 bool 1100 ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, 1101 SDValue &OffImm) { 1102 return SelectThumbAddrModeImm5S(N, 2, Base, OffImm); 1103 } 1104 1105 bool 1106 ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, 1107 SDValue &OffImm) { 1108 return SelectThumbAddrModeImm5S(N, 1, Base, OffImm); 1109 } 1110 1111 bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N, 1112 SDValue &Base, SDValue &OffImm) { 1113 if (N.getOpcode() == ISD::FrameIndex) { 1114 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 1115 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 1116 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1117 return true; 1118 } 1119 1120 if (!CurDAG->isBaseWithConstantOffset(N)) 1121 return false; 1122 1123 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 1124 if (N.getOperand(0).getOpcode() == ISD::FrameIndex || 1125 (LHSR && LHSR->getReg() == ARM::SP)) { 1126 // If the RHS is + imm8 * scale, fold into addr mode. 1127 int RHSC; 1128 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) { 1129 Base = N.getOperand(0); 1130 if (Base.getOpcode() == ISD::FrameIndex) { 1131 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1132 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 1133 } 1134 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1135 return true; 1136 } 1137 } 1138 1139 return false; 1140 } 1141 1142 1143 //===----------------------------------------------------------------------===// 1144 // Thumb 2 Addressing Modes 1145 //===----------------------------------------------------------------------===// 1146 1147 1148 bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg, 1149 SDValue &Opc) { 1150 if (DisableShifterOp) 1151 return false; 1152 1153 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 1154 1155 // Don't match base register only case. That is matched to a separate 1156 // lower complexity pattern with explicit register operand. 1157 if (ShOpcVal == ARM_AM::no_shift) return false; 1158 1159 BaseReg = N.getOperand(0); 1160 unsigned ShImmVal = 0; 1161 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1162 ShImmVal = RHS->getZExtValue() & 31; 1163 Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal)); 1164 return true; 1165 } 1166 1167 return false; 1168 } 1169 1170 bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N, 1171 SDValue &Base, SDValue &OffImm) { 1172 // Match simple R + imm12 operands. 1173 1174 // Base only. 1175 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 1176 !CurDAG->isBaseWithConstantOffset(N)) { 1177 if (N.getOpcode() == ISD::FrameIndex) { 1178 // Match frame index. 1179 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 1180 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 1181 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1182 return true; 1183 } 1184 1185 if (N.getOpcode() == ARMISD::Wrapper && 1186 !(Subtarget->useMovt() && 1187 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { 1188 Base = N.getOperand(0); 1189 if (Base.getOpcode() == ISD::TargetConstantPool) 1190 return false; // We want to select t2LDRpci instead. 1191 } else 1192 Base = N; 1193 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1194 return true; 1195 } 1196 1197 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1198 if (SelectT2AddrModeImm8(N, Base, OffImm)) 1199 // Let t2LDRi8 handle (R - imm8). 1200 return false; 1201 1202 int RHSC = (int)RHS->getZExtValue(); 1203 if (N.getOpcode() == ISD::SUB) 1204 RHSC = -RHSC; 1205 1206 if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned) 1207 Base = N.getOperand(0); 1208 if (Base.getOpcode() == ISD::FrameIndex) { 1209 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1210 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 1211 } 1212 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1213 return true; 1214 } 1215 } 1216 1217 // Base only. 1218 Base = N; 1219 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1220 return true; 1221 } 1222 1223 bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N, 1224 SDValue &Base, SDValue &OffImm) { 1225 // Match simple R - imm8 operands. 1226 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 1227 !CurDAG->isBaseWithConstantOffset(N)) 1228 return false; 1229 1230 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1231 int RHSC = (int)RHS->getSExtValue(); 1232 if (N.getOpcode() == ISD::SUB) 1233 RHSC = -RHSC; 1234 1235 if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative) 1236 Base = N.getOperand(0); 1237 if (Base.getOpcode() == ISD::FrameIndex) { 1238 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1239 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 1240 } 1241 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1242 return true; 1243 } 1244 } 1245 1246 return false; 1247 } 1248 1249 bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, 1250 SDValue &OffImm){ 1251 unsigned Opcode = Op->getOpcode(); 1252 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 1253 ? cast<LoadSDNode>(Op)->getAddressingMode() 1254 : cast<StoreSDNode>(Op)->getAddressingMode(); 1255 int RHSC; 1256 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits. 1257 OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC)) 1258 ? CurDAG->getTargetConstant(RHSC, MVT::i32) 1259 : CurDAG->getTargetConstant(-RHSC, MVT::i32); 1260 return true; 1261 } 1262 1263 return false; 1264 } 1265 1266 bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N, 1267 SDValue &Base, 1268 SDValue &OffReg, SDValue &ShImm) { 1269 // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12. 1270 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) 1271 return false; 1272 1273 // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8. 1274 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1275 int RHSC = (int)RHS->getZExtValue(); 1276 if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned) 1277 return false; 1278 else if (RHSC < 0 && RHSC >= -255) // 8 bits 1279 return false; 1280 } 1281 1282 // Look for (R + R) or (R + (R << [1,2,3])). 1283 unsigned ShAmt = 0; 1284 Base = N.getOperand(0); 1285 OffReg = N.getOperand(1); 1286 1287 // Swap if it is ((R << c) + R). 1288 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode()); 1289 if (ShOpcVal != ARM_AM::lsl) { 1290 ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode()); 1291 if (ShOpcVal == ARM_AM::lsl) 1292 std::swap(Base, OffReg); 1293 } 1294 1295 if (ShOpcVal == ARM_AM::lsl) { 1296 // Check to see if the RHS of the shift is a constant, if not, we can't fold 1297 // it. 1298 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) { 1299 ShAmt = Sh->getZExtValue(); 1300 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt)) 1301 OffReg = OffReg.getOperand(0); 1302 else { 1303 ShAmt = 0; 1304 ShOpcVal = ARM_AM::no_shift; 1305 } 1306 } else { 1307 ShOpcVal = ARM_AM::no_shift; 1308 } 1309 } 1310 1311 ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32); 1312 1313 return true; 1314 } 1315 1316 //===--------------------------------------------------------------------===// 1317 1318 /// getAL - Returns a ARMCC::AL immediate node. 1319 static inline SDValue getAL(SelectionDAG *CurDAG) { 1320 return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32); 1321 } 1322 1323 SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) { 1324 LoadSDNode *LD = cast<LoadSDNode>(N); 1325 ISD::MemIndexedMode AM = LD->getAddressingMode(); 1326 if (AM == ISD::UNINDEXED) 1327 return NULL; 1328 1329 EVT LoadedVT = LD->getMemoryVT(); 1330 SDValue Offset, AMOpc; 1331 bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); 1332 unsigned Opcode = 0; 1333 bool Match = false; 1334 if (LoadedVT == MVT::i32 && isPre && 1335 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { 1336 Opcode = ARM::LDR_PRE_IMM; 1337 Match = true; 1338 } else if (LoadedVT == MVT::i32 && !isPre && 1339 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { 1340 Opcode = ARM::LDR_POST_IMM; 1341 Match = true; 1342 } else if (LoadedVT == MVT::i32 && 1343 SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { 1344 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG; 1345 Match = true; 1346 1347 } else if (LoadedVT == MVT::i16 && 1348 SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { 1349 Match = true; 1350 Opcode = (LD->getExtensionType() == ISD::SEXTLOAD) 1351 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST) 1352 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST); 1353 } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) { 1354 if (LD->getExtensionType() == ISD::SEXTLOAD) { 1355 if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { 1356 Match = true; 1357 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST; 1358 } 1359 } else { 1360 if (isPre && 1361 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { 1362 Match = true; 1363 Opcode = ARM::LDRB_PRE_IMM; 1364 } else if (!isPre && 1365 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { 1366 Match = true; 1367 Opcode = ARM::LDRB_POST_IMM; 1368 } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { 1369 Match = true; 1370 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG; 1371 } 1372 } 1373 } 1374 1375 if (Match) { 1376 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) { 1377 SDValue Chain = LD->getChain(); 1378 SDValue Base = LD->getBasePtr(); 1379 SDValue Ops[]= { Base, AMOpc, getAL(CurDAG), 1380 CurDAG->getRegister(0, MVT::i32), Chain }; 1381 return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, 1382 MVT::i32, MVT::Other, Ops, 5); 1383 } else { 1384 SDValue Chain = LD->getChain(); 1385 SDValue Base = LD->getBasePtr(); 1386 SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG), 1387 CurDAG->getRegister(0, MVT::i32), Chain }; 1388 return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, 1389 MVT::i32, MVT::Other, Ops, 6); 1390 } 1391 } 1392 1393 return NULL; 1394 } 1395 1396 SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) { 1397 LoadSDNode *LD = cast<LoadSDNode>(N); 1398 ISD::MemIndexedMode AM = LD->getAddressingMode(); 1399 if (AM == ISD::UNINDEXED) 1400 return NULL; 1401 1402 EVT LoadedVT = LD->getMemoryVT(); 1403 bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD; 1404 SDValue Offset; 1405 bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); 1406 unsigned Opcode = 0; 1407 bool Match = false; 1408 if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) { 1409 switch (LoadedVT.getSimpleVT().SimpleTy) { 1410 case MVT::i32: 1411 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST; 1412 break; 1413 case MVT::i16: 1414 if (isSExtLd) 1415 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST; 1416 else 1417 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST; 1418 break; 1419 case MVT::i8: 1420 case MVT::i1: 1421 if (isSExtLd) 1422 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST; 1423 else 1424 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST; 1425 break; 1426 default: 1427 return NULL; 1428 } 1429 Match = true; 1430 } 1431 1432 if (Match) { 1433 SDValue Chain = LD->getChain(); 1434 SDValue Base = LD->getBasePtr(); 1435 SDValue Ops[]= { Base, Offset, getAL(CurDAG), 1436 CurDAG->getRegister(0, MVT::i32), Chain }; 1437 return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32, 1438 MVT::Other, Ops, 5); 1439 } 1440 1441 return NULL; 1442 } 1443 1444 /// PairSRegs - Form a D register from a pair of S registers. 1445 /// 1446 SDNode *ARMDAGToDAGISel::PairSRegs(EVT VT, SDValue V0, SDValue V1) { 1447 DebugLoc dl = V0.getNode()->getDebugLoc(); 1448 SDValue RegClass = 1449 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32); 1450 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32); 1451 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32); 1452 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1453 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5); 1454 } 1455 1456 /// PairDRegs - Form a quad register from a pair of D registers. 1457 /// 1458 SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) { 1459 DebugLoc dl = V0.getNode()->getDebugLoc(); 1460 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32); 1461 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32); 1462 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32); 1463 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1464 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5); 1465 } 1466 1467 /// PairQRegs - Form 4 consecutive D registers from a pair of Q registers. 1468 /// 1469 SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) { 1470 DebugLoc dl = V0.getNode()->getDebugLoc(); 1471 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32); 1472 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32); 1473 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32); 1474 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1475 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5); 1476 } 1477 1478 /// QuadSRegs - Form 4 consecutive S registers. 1479 /// 1480 SDNode *ARMDAGToDAGISel::QuadSRegs(EVT VT, SDValue V0, SDValue V1, 1481 SDValue V2, SDValue V3) { 1482 DebugLoc dl = V0.getNode()->getDebugLoc(); 1483 SDValue RegClass = 1484 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32); 1485 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32); 1486 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32); 1487 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32); 1488 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32); 1489 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1490 V2, SubReg2, V3, SubReg3 }; 1491 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9); 1492 } 1493 1494 /// QuadDRegs - Form 4 consecutive D registers. 1495 /// 1496 SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1, 1497 SDValue V2, SDValue V3) { 1498 DebugLoc dl = V0.getNode()->getDebugLoc(); 1499 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32); 1500 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32); 1501 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32); 1502 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32); 1503 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32); 1504 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1505 V2, SubReg2, V3, SubReg3 }; 1506 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9); 1507 } 1508 1509 /// QuadQRegs - Form 4 consecutive Q registers. 1510 /// 1511 SDNode *ARMDAGToDAGISel::QuadQRegs(EVT VT, SDValue V0, SDValue V1, 1512 SDValue V2, SDValue V3) { 1513 DebugLoc dl = V0.getNode()->getDebugLoc(); 1514 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32); 1515 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32); 1516 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32); 1517 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32); 1518 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32); 1519 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1520 V2, SubReg2, V3, SubReg3 }; 1521 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9); 1522 } 1523 1524 /// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand 1525 /// of a NEON VLD or VST instruction. The supported values depend on the 1526 /// number of registers being loaded. 1527 SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs, 1528 bool is64BitVector) { 1529 unsigned NumRegs = NumVecs; 1530 if (!is64BitVector && NumVecs < 3) 1531 NumRegs *= 2; 1532 1533 unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 1534 if (Alignment >= 32 && NumRegs == 4) 1535 Alignment = 32; 1536 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4)) 1537 Alignment = 16; 1538 else if (Alignment >= 8) 1539 Alignment = 8; 1540 else 1541 Alignment = 0; 1542 1543 return CurDAG->getTargetConstant(Alignment, MVT::i32); 1544 } 1545 1546 // Get the register stride update opcode of a VLD/VST instruction that 1547 // is otherwise equivalent to the given fixed stride updating instruction. 1548 static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) { 1549 switch (Opc) { 1550 default: break; 1551 case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register; 1552 case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register; 1553 case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register; 1554 case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register; 1555 case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register; 1556 case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register; 1557 case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register; 1558 case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register; 1559 1560 case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register; 1561 case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register; 1562 case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register; 1563 case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register; 1564 case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register; 1565 case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register; 1566 case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register; 1567 case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register; 1568 case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register; 1569 case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register; 1570 1571 case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register; 1572 case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register; 1573 case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register; 1574 case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register; 1575 case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register; 1576 case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register; 1577 1578 case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register; 1579 case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register; 1580 case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register; 1581 case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register; 1582 case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register; 1583 case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register; 1584 1585 case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register; 1586 case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register; 1587 case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register; 1588 } 1589 return Opc; // If not one we handle, return it unchanged. 1590 } 1591 1592 SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, 1593 const uint16_t *DOpcodes, 1594 const uint16_t *QOpcodes0, 1595 const uint16_t *QOpcodes1) { 1596 assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range"); 1597 DebugLoc dl = N->getDebugLoc(); 1598 1599 SDValue MemAddr, Align; 1600 unsigned AddrOpIdx = isUpdating ? 1 : 2; 1601 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 1602 return NULL; 1603 1604 SDValue Chain = N->getOperand(0); 1605 EVT VT = N->getValueType(0); 1606 bool is64BitVector = VT.is64BitVector(); 1607 Align = GetVLDSTAlign(Align, NumVecs, is64BitVector); 1608 1609 unsigned OpcodeIndex; 1610 switch (VT.getSimpleVT().SimpleTy) { 1611 default: llvm_unreachable("unhandled vld type"); 1612 // Double-register operations: 1613 case MVT::v8i8: OpcodeIndex = 0; break; 1614 case MVT::v4i16: OpcodeIndex = 1; break; 1615 case MVT::v2f32: 1616 case MVT::v2i32: OpcodeIndex = 2; break; 1617 case MVT::v1i64: OpcodeIndex = 3; break; 1618 // Quad-register operations: 1619 case MVT::v16i8: OpcodeIndex = 0; break; 1620 case MVT::v8i16: OpcodeIndex = 1; break; 1621 case MVT::v4f32: 1622 case MVT::v4i32: OpcodeIndex = 2; break; 1623 case MVT::v2i64: OpcodeIndex = 3; 1624 assert(NumVecs == 1 && "v2i64 type only supported for VLD1"); 1625 break; 1626 } 1627 1628 EVT ResTy; 1629 if (NumVecs == 1) 1630 ResTy = VT; 1631 else { 1632 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 1633 if (!is64BitVector) 1634 ResTyElts *= 2; 1635 ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts); 1636 } 1637 std::vector<EVT> ResTys; 1638 ResTys.push_back(ResTy); 1639 if (isUpdating) 1640 ResTys.push_back(MVT::i32); 1641 ResTys.push_back(MVT::Other); 1642 1643 SDValue Pred = getAL(CurDAG); 1644 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 1645 SDNode *VLd; 1646 SmallVector<SDValue, 7> Ops; 1647 1648 // Double registers and VLD1/VLD2 quad registers are directly supported. 1649 if (is64BitVector || NumVecs <= 2) { 1650 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 1651 QOpcodes0[OpcodeIndex]); 1652 Ops.push_back(MemAddr); 1653 Ops.push_back(Align); 1654 if (isUpdating) { 1655 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1656 // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0 1657 // case entirely when the rest are updated to that form, too. 1658 if ((NumVecs == 1 || NumVecs == 2) && !isa<ConstantSDNode>(Inc.getNode())) 1659 Opc = getVLDSTRegisterUpdateOpcode(Opc); 1660 // We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so 1661 // check for that explicitly too. Horribly hacky, but temporary. 1662 if ((NumVecs != 1 && NumVecs != 2 && Opc != ARM::VLD1q64wb_fixed) || 1663 !isa<ConstantSDNode>(Inc.getNode())) 1664 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); 1665 } 1666 Ops.push_back(Pred); 1667 Ops.push_back(Reg0); 1668 Ops.push_back(Chain); 1669 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size()); 1670 1671 } else { 1672 // Otherwise, quad registers are loaded with two separate instructions, 1673 // where one loads the even registers and the other loads the odd registers. 1674 EVT AddrTy = MemAddr.getValueType(); 1675 1676 // Load the even subregs. This is always an updating load, so that it 1677 // provides the address to the second load for the odd subregs. 1678 SDValue ImplDef = 1679 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0); 1680 const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain }; 1681 SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, 1682 ResTy, AddrTy, MVT::Other, OpsA, 7); 1683 Chain = SDValue(VLdA, 2); 1684 1685 // Load the odd subregs. 1686 Ops.push_back(SDValue(VLdA, 1)); 1687 Ops.push_back(Align); 1688 if (isUpdating) { 1689 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1690 assert(isa<ConstantSDNode>(Inc.getNode()) && 1691 "only constant post-increment update allowed for VLD3/4"); 1692 (void)Inc; 1693 Ops.push_back(Reg0); 1694 } 1695 Ops.push_back(SDValue(VLdA, 0)); 1696 Ops.push_back(Pred); 1697 Ops.push_back(Reg0); 1698 Ops.push_back(Chain); 1699 VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, 1700 Ops.data(), Ops.size()); 1701 } 1702 1703 // Transfer memoperands. 1704 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1705 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 1706 cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1); 1707 1708 if (NumVecs == 1) 1709 return VLd; 1710 1711 // Extract out the subregisters. 1712 SDValue SuperReg = SDValue(VLd, 0); 1713 assert(ARM::dsub_7 == ARM::dsub_0+7 && 1714 ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering"); 1715 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0); 1716 for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 1717 ReplaceUses(SDValue(N, Vec), 1718 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); 1719 ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1)); 1720 if (isUpdating) 1721 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2)); 1722 return NULL; 1723 } 1724 1725 SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, 1726 const uint16_t *DOpcodes, 1727 const uint16_t *QOpcodes0, 1728 const uint16_t *QOpcodes1) { 1729 assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range"); 1730 DebugLoc dl = N->getDebugLoc(); 1731 1732 SDValue MemAddr, Align; 1733 unsigned AddrOpIdx = isUpdating ? 1 : 2; 1734 unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1) 1735 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 1736 return NULL; 1737 1738 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1739 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 1740 1741 SDValue Chain = N->getOperand(0); 1742 EVT VT = N->getOperand(Vec0Idx).getValueType(); 1743 bool is64BitVector = VT.is64BitVector(); 1744 Align = GetVLDSTAlign(Align, NumVecs, is64BitVector); 1745 1746 unsigned OpcodeIndex; 1747 switch (VT.getSimpleVT().SimpleTy) { 1748 default: llvm_unreachable("unhandled vst type"); 1749 // Double-register operations: 1750 case MVT::v8i8: OpcodeIndex = 0; break; 1751 case MVT::v4i16: OpcodeIndex = 1; break; 1752 case MVT::v2f32: 1753 case MVT::v2i32: OpcodeIndex = 2; break; 1754 case MVT::v1i64: OpcodeIndex = 3; break; 1755 // Quad-register operations: 1756 case MVT::v16i8: OpcodeIndex = 0; break; 1757 case MVT::v8i16: OpcodeIndex = 1; break; 1758 case MVT::v4f32: 1759 case MVT::v4i32: OpcodeIndex = 2; break; 1760 case MVT::v2i64: OpcodeIndex = 3; 1761 assert(NumVecs == 1 && "v2i64 type only supported for VST1"); 1762 break; 1763 } 1764 1765 std::vector<EVT> ResTys; 1766 if (isUpdating) 1767 ResTys.push_back(MVT::i32); 1768 ResTys.push_back(MVT::Other); 1769 1770 SDValue Pred = getAL(CurDAG); 1771 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 1772 SmallVector<SDValue, 7> Ops; 1773 1774 // Double registers and VST1/VST2 quad registers are directly supported. 1775 if (is64BitVector || NumVecs <= 2) { 1776 SDValue SrcReg; 1777 if (NumVecs == 1) { 1778 SrcReg = N->getOperand(Vec0Idx); 1779 } else if (is64BitVector) { 1780 // Form a REG_SEQUENCE to force register allocation. 1781 SDValue V0 = N->getOperand(Vec0Idx + 0); 1782 SDValue V1 = N->getOperand(Vec0Idx + 1); 1783 if (NumVecs == 2) 1784 SrcReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0); 1785 else { 1786 SDValue V2 = N->getOperand(Vec0Idx + 2); 1787 // If it's a vst3, form a quad D-register and leave the last part as 1788 // an undef. 1789 SDValue V3 = (NumVecs == 3) 1790 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0) 1791 : N->getOperand(Vec0Idx + 3); 1792 SrcReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0); 1793 } 1794 } else { 1795 // Form a QQ register. 1796 SDValue Q0 = N->getOperand(Vec0Idx); 1797 SDValue Q1 = N->getOperand(Vec0Idx + 1); 1798 SrcReg = SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0); 1799 } 1800 1801 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 1802 QOpcodes0[OpcodeIndex]); 1803 Ops.push_back(MemAddr); 1804 Ops.push_back(Align); 1805 if (isUpdating) { 1806 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1807 // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0 1808 // case entirely when the rest are updated to that form, too. 1809 if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode())) 1810 Opc = getVLDSTRegisterUpdateOpcode(Opc); 1811 // We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so 1812 // check for that explicitly too. Horribly hacky, but temporary. 1813 if ((NumVecs > 2 && Opc != ARM::VST1q64wb_fixed) || 1814 !isa<ConstantSDNode>(Inc.getNode())) 1815 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); 1816 } 1817 Ops.push_back(SrcReg); 1818 Ops.push_back(Pred); 1819 Ops.push_back(Reg0); 1820 Ops.push_back(Chain); 1821 SDNode *VSt = 1822 CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size()); 1823 1824 // Transfer memoperands. 1825 cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1); 1826 1827 return VSt; 1828 } 1829 1830 // Otherwise, quad registers are stored with two separate instructions, 1831 // where one stores the even registers and the other stores the odd registers. 1832 1833 // Form the QQQQ REG_SEQUENCE. 1834 SDValue V0 = N->getOperand(Vec0Idx + 0); 1835 SDValue V1 = N->getOperand(Vec0Idx + 1); 1836 SDValue V2 = N->getOperand(Vec0Idx + 2); 1837 SDValue V3 = (NumVecs == 3) 1838 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 1839 : N->getOperand(Vec0Idx + 3); 1840 SDValue RegSeq = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0); 1841 1842 // Store the even D registers. This is always an updating store, so that it 1843 // provides the address to the second store for the odd subregs. 1844 const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain }; 1845 SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, 1846 MemAddr.getValueType(), 1847 MVT::Other, OpsA, 7); 1848 cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1); 1849 Chain = SDValue(VStA, 1); 1850 1851 // Store the odd D registers. 1852 Ops.push_back(SDValue(VStA, 0)); 1853 Ops.push_back(Align); 1854 if (isUpdating) { 1855 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1856 assert(isa<ConstantSDNode>(Inc.getNode()) && 1857 "only constant post-increment update allowed for VST3/4"); 1858 (void)Inc; 1859 Ops.push_back(Reg0); 1860 } 1861 Ops.push_back(RegSeq); 1862 Ops.push_back(Pred); 1863 Ops.push_back(Reg0); 1864 Ops.push_back(Chain); 1865 SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, 1866 Ops.data(), Ops.size()); 1867 cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1); 1868 return VStB; 1869 } 1870 1871 SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, 1872 bool isUpdating, unsigned NumVecs, 1873 const uint16_t *DOpcodes, 1874 const uint16_t *QOpcodes) { 1875 assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range"); 1876 DebugLoc dl = N->getDebugLoc(); 1877 1878 SDValue MemAddr, Align; 1879 unsigned AddrOpIdx = isUpdating ? 1 : 2; 1880 unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1) 1881 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 1882 return NULL; 1883 1884 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1885 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 1886 1887 SDValue Chain = N->getOperand(0); 1888 unsigned Lane = 1889 cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue(); 1890 EVT VT = N->getOperand(Vec0Idx).getValueType(); 1891 bool is64BitVector = VT.is64BitVector(); 1892 1893 unsigned Alignment = 0; 1894 if (NumVecs != 3) { 1895 Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 1896 unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8; 1897 if (Alignment > NumBytes) 1898 Alignment = NumBytes; 1899 if (Alignment < 8 && Alignment < NumBytes) 1900 Alignment = 0; 1901 // Alignment must be a power of two; make sure of that. 1902 Alignment = (Alignment & -Alignment); 1903 if (Alignment == 1) 1904 Alignment = 0; 1905 } 1906 Align = CurDAG->getTargetConstant(Alignment, MVT::i32); 1907 1908 unsigned OpcodeIndex; 1909 switch (VT.getSimpleVT().SimpleTy) { 1910 default: llvm_unreachable("unhandled vld/vst lane type"); 1911 // Double-register operations: 1912 case MVT::v8i8: OpcodeIndex = 0; break; 1913 case MVT::v4i16: OpcodeIndex = 1; break; 1914 case MVT::v2f32: 1915 case MVT::v2i32: OpcodeIndex = 2; break; 1916 // Quad-register operations: 1917 case MVT::v8i16: OpcodeIndex = 0; break; 1918 case MVT::v4f32: 1919 case MVT::v4i32: OpcodeIndex = 1; break; 1920 } 1921 1922 std::vector<EVT> ResTys; 1923 if (IsLoad) { 1924 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 1925 if (!is64BitVector) 1926 ResTyElts *= 2; 1927 ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), 1928 MVT::i64, ResTyElts)); 1929 } 1930 if (isUpdating) 1931 ResTys.push_back(MVT::i32); 1932 ResTys.push_back(MVT::Other); 1933 1934 SDValue Pred = getAL(CurDAG); 1935 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 1936 1937 SmallVector<SDValue, 8> Ops; 1938 Ops.push_back(MemAddr); 1939 Ops.push_back(Align); 1940 if (isUpdating) { 1941 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1942 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); 1943 } 1944 1945 SDValue SuperReg; 1946 SDValue V0 = N->getOperand(Vec0Idx + 0); 1947 SDValue V1 = N->getOperand(Vec0Idx + 1); 1948 if (NumVecs == 2) { 1949 if (is64BitVector) 1950 SuperReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0); 1951 else 1952 SuperReg = SDValue(PairQRegs(MVT::v4i64, V0, V1), 0); 1953 } else { 1954 SDValue V2 = N->getOperand(Vec0Idx + 2); 1955 SDValue V3 = (NumVecs == 3) 1956 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 1957 : N->getOperand(Vec0Idx + 3); 1958 if (is64BitVector) 1959 SuperReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0); 1960 else 1961 SuperReg = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0); 1962 } 1963 Ops.push_back(SuperReg); 1964 Ops.push_back(getI32Imm(Lane)); 1965 Ops.push_back(Pred); 1966 Ops.push_back(Reg0); 1967 Ops.push_back(Chain); 1968 1969 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 1970 QOpcodes[OpcodeIndex]); 1971 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, 1972 Ops.data(), Ops.size()); 1973 cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1); 1974 if (!IsLoad) 1975 return VLdLn; 1976 1977 // Extract the subregisters. 1978 SuperReg = SDValue(VLdLn, 0); 1979 assert(ARM::dsub_7 == ARM::dsub_0+7 && 1980 ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering"); 1981 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0; 1982 for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 1983 ReplaceUses(SDValue(N, Vec), 1984 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); 1985 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1)); 1986 if (isUpdating) 1987 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2)); 1988 return NULL; 1989 } 1990 1991 SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating, 1992 unsigned NumVecs, 1993 const uint16_t *Opcodes) { 1994 assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range"); 1995 DebugLoc dl = N->getDebugLoc(); 1996 1997 SDValue MemAddr, Align; 1998 if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align)) 1999 return NULL; 2000 2001 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 2002 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 2003 2004 SDValue Chain = N->getOperand(0); 2005 EVT VT = N->getValueType(0); 2006 2007 unsigned Alignment = 0; 2008 if (NumVecs != 3) { 2009 Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 2010 unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8; 2011 if (Alignment > NumBytes) 2012 Alignment = NumBytes; 2013 if (Alignment < 8 && Alignment < NumBytes) 2014 Alignment = 0; 2015 // Alignment must be a power of two; make sure of that. 2016 Alignment = (Alignment & -Alignment); 2017 if (Alignment == 1) 2018 Alignment = 0; 2019 } 2020 Align = CurDAG->getTargetConstant(Alignment, MVT::i32); 2021 2022 unsigned OpcodeIndex; 2023 switch (VT.getSimpleVT().SimpleTy) { 2024 default: llvm_unreachable("unhandled vld-dup type"); 2025 case MVT::v8i8: OpcodeIndex = 0; break; 2026 case MVT::v4i16: OpcodeIndex = 1; break; 2027 case MVT::v2f32: 2028 case MVT::v2i32: OpcodeIndex = 2; break; 2029 } 2030 2031 SDValue Pred = getAL(CurDAG); 2032 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2033 SDValue SuperReg; 2034 unsigned Opc = Opcodes[OpcodeIndex]; 2035 SmallVector<SDValue, 6> Ops; 2036 Ops.push_back(MemAddr); 2037 Ops.push_back(Align); 2038 if (isUpdating) { 2039 // fixed-stride update instructions don't have an explicit writeback 2040 // operand. It's implicit in the opcode itself. 2041 SDValue Inc = N->getOperand(2); 2042 if (!isa<ConstantSDNode>(Inc.getNode())) 2043 Ops.push_back(Inc); 2044 // FIXME: VLD3 and VLD4 haven't been updated to that form yet. 2045 else if (NumVecs > 2) 2046 Ops.push_back(Reg0); 2047 } 2048 Ops.push_back(Pred); 2049 Ops.push_back(Reg0); 2050 Ops.push_back(Chain); 2051 2052 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 2053 std::vector<EVT> ResTys; 2054 ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts)); 2055 if (isUpdating) 2056 ResTys.push_back(MVT::i32); 2057 ResTys.push_back(MVT::Other); 2058 SDNode *VLdDup = 2059 CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size()); 2060 cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1); 2061 SuperReg = SDValue(VLdDup, 0); 2062 2063 // Extract the subregisters. 2064 assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering"); 2065 unsigned SubIdx = ARM::dsub_0; 2066 for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 2067 ReplaceUses(SDValue(N, Vec), 2068 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg)); 2069 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1)); 2070 if (isUpdating) 2071 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2)); 2072 return NULL; 2073 } 2074 2075 SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, 2076 unsigned Opc) { 2077 assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range"); 2078 DebugLoc dl = N->getDebugLoc(); 2079 EVT VT = N->getValueType(0); 2080 unsigned FirstTblReg = IsExt ? 2 : 1; 2081 2082 // Form a REG_SEQUENCE to force register allocation. 2083 SDValue RegSeq; 2084 SDValue V0 = N->getOperand(FirstTblReg + 0); 2085 SDValue V1 = N->getOperand(FirstTblReg + 1); 2086 if (NumVecs == 2) 2087 RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0); 2088 else { 2089 SDValue V2 = N->getOperand(FirstTblReg + 2); 2090 // If it's a vtbl3, form a quad D-register and leave the last part as 2091 // an undef. 2092 SDValue V3 = (NumVecs == 3) 2093 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 2094 : N->getOperand(FirstTblReg + 3); 2095 RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0); 2096 } 2097 2098 SmallVector<SDValue, 6> Ops; 2099 if (IsExt) 2100 Ops.push_back(N->getOperand(1)); 2101 Ops.push_back(RegSeq); 2102 Ops.push_back(N->getOperand(FirstTblReg + NumVecs)); 2103 Ops.push_back(getAL(CurDAG)); // predicate 2104 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register 2105 return CurDAG->getMachineNode(Opc, dl, VT, Ops.data(), Ops.size()); 2106 } 2107 2108 SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N, 2109 bool isSigned) { 2110 if (!Subtarget->hasV6T2Ops()) 2111 return NULL; 2112 2113 unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX) 2114 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX); 2115 2116 2117 // For unsigned extracts, check for a shift right and mask 2118 unsigned And_imm = 0; 2119 if (N->getOpcode() == ISD::AND) { 2120 if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) { 2121 2122 // The immediate is a mask of the low bits iff imm & (imm+1) == 0 2123 if (And_imm & (And_imm + 1)) 2124 return NULL; 2125 2126 unsigned Srl_imm = 0; 2127 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, 2128 Srl_imm)) { 2129 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); 2130 2131 // Note: The width operand is encoded as width-1. 2132 unsigned Width = CountTrailingOnes_32(And_imm) - 1; 2133 unsigned LSB = Srl_imm; 2134 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2135 SDValue Ops[] = { N->getOperand(0).getOperand(0), 2136 CurDAG->getTargetConstant(LSB, MVT::i32), 2137 CurDAG->getTargetConstant(Width, MVT::i32), 2138 getAL(CurDAG), Reg0 }; 2139 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); 2140 } 2141 } 2142 return NULL; 2143 } 2144 2145 // Otherwise, we're looking for a shift of a shift 2146 unsigned Shl_imm = 0; 2147 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) { 2148 assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!"); 2149 unsigned Srl_imm = 0; 2150 if (isInt32Immediate(N->getOperand(1), Srl_imm)) { 2151 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); 2152 // Note: The width operand is encoded as width-1. 2153 unsigned Width = 32 - Srl_imm - 1; 2154 int LSB = Srl_imm - Shl_imm; 2155 if (LSB < 0) 2156 return NULL; 2157 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2158 SDValue Ops[] = { N->getOperand(0).getOperand(0), 2159 CurDAG->getTargetConstant(LSB, MVT::i32), 2160 CurDAG->getTargetConstant(Width, MVT::i32), 2161 getAL(CurDAG), Reg0 }; 2162 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); 2163 } 2164 } 2165 return NULL; 2166 } 2167 2168 SDNode *ARMDAGToDAGISel:: 2169 SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 2170 ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) { 2171 SDValue CPTmp0; 2172 SDValue CPTmp1; 2173 if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) { 2174 unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue(); 2175 unsigned SOShOp = ARM_AM::getSORegShOp(SOVal); 2176 unsigned Opc = 0; 2177 switch (SOShOp) { 2178 case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break; 2179 case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break; 2180 case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break; 2181 case ARM_AM::ror: Opc = ARM::t2MOVCCror; break; 2182 default: 2183 llvm_unreachable("Unknown so_reg opcode!"); 2184 } 2185 SDValue SOShImm = 2186 CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32); 2187 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); 2188 SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag }; 2189 return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6); 2190 } 2191 return 0; 2192 } 2193 2194 SDNode *ARMDAGToDAGISel:: 2195 SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 2196 ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) { 2197 SDValue CPTmp0; 2198 SDValue CPTmp1; 2199 SDValue CPTmp2; 2200 if (SelectImmShifterOperand(TrueVal, CPTmp0, CPTmp2)) { 2201 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); 2202 SDValue Ops[] = { FalseVal, CPTmp0, CPTmp2, CC, CCR, InFlag }; 2203 return CurDAG->SelectNodeTo(N, ARM::MOVCCsi, MVT::i32, Ops, 6); 2204 } 2205 2206 if (SelectRegShifterOperand(TrueVal, CPTmp0, CPTmp1, CPTmp2)) { 2207 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); 2208 SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag }; 2209 return CurDAG->SelectNodeTo(N, ARM::MOVCCsr, MVT::i32, Ops, 7); 2210 } 2211 return 0; 2212 } 2213 2214 SDNode *ARMDAGToDAGISel:: 2215 SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 2216 ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) { 2217 ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal); 2218 if (!T) 2219 return 0; 2220 2221 unsigned Opc = 0; 2222 unsigned TrueImm = T->getZExtValue(); 2223 if (is_t2_so_imm(TrueImm)) { 2224 Opc = ARM::t2MOVCCi; 2225 } else if (TrueImm <= 0xffff) { 2226 Opc = ARM::t2MOVCCi16; 2227 } else if (is_t2_so_imm_not(TrueImm)) { 2228 TrueImm = ~TrueImm; 2229 Opc = ARM::t2MVNCCi; 2230 } else if (TrueVal.getNode()->hasOneUse() && Subtarget->hasV6T2Ops()) { 2231 // Large immediate. 2232 Opc = ARM::t2MOVCCi32imm; 2233 } 2234 2235 if (Opc) { 2236 SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32); 2237 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); 2238 SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag }; 2239 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); 2240 } 2241 2242 return 0; 2243 } 2244 2245 SDNode *ARMDAGToDAGISel:: 2246 SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 2247 ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) { 2248 ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal); 2249 if (!T) 2250 return 0; 2251 2252 unsigned Opc = 0; 2253 unsigned TrueImm = T->getZExtValue(); 2254 bool isSoImm = is_so_imm(TrueImm); 2255 if (isSoImm) { 2256 Opc = ARM::MOVCCi; 2257 } else if (Subtarget->hasV6T2Ops() && TrueImm <= 0xffff) { 2258 Opc = ARM::MOVCCi16; 2259 } else if (is_so_imm_not(TrueImm)) { 2260 TrueImm = ~TrueImm; 2261 Opc = ARM::MVNCCi; 2262 } else if (TrueVal.getNode()->hasOneUse() && 2263 (Subtarget->hasV6T2Ops() || ARM_AM::isSOImmTwoPartVal(TrueImm))) { 2264 // Large immediate. 2265 Opc = ARM::MOVCCi32imm; 2266 } 2267 2268 if (Opc) { 2269 SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32); 2270 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); 2271 SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag }; 2272 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); 2273 } 2274 2275 return 0; 2276 } 2277 2278 SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) { 2279 EVT VT = N->getValueType(0); 2280 SDValue FalseVal = N->getOperand(0); 2281 SDValue TrueVal = N->getOperand(1); 2282 SDValue CC = N->getOperand(2); 2283 SDValue CCR = N->getOperand(3); 2284 SDValue InFlag = N->getOperand(4); 2285 assert(CC.getOpcode() == ISD::Constant); 2286 assert(CCR.getOpcode() == ISD::Register); 2287 ARMCC::CondCodes CCVal = 2288 (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue(); 2289 2290 if (!Subtarget->isThumb1Only() && VT == MVT::i32) { 2291 // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc) 2292 // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc) 2293 // Pattern complexity = 18 cost = 1 size = 0 2294 if (Subtarget->isThumb()) { 2295 SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal, 2296 CCVal, CCR, InFlag); 2297 if (!Res) 2298 Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal, 2299 ARMCC::getOppositeCondition(CCVal), CCR, InFlag); 2300 if (Res) 2301 return Res; 2302 } else { 2303 SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal, 2304 CCVal, CCR, InFlag); 2305 if (!Res) 2306 Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal, 2307 ARMCC::getOppositeCondition(CCVal), CCR, InFlag); 2308 if (Res) 2309 return Res; 2310 } 2311 2312 // Pattern: (ARMcmov:i32 GPR:i32:$false, 2313 // (imm:i32)<<P:Pred_so_imm>>:$true, 2314 // (imm:i32):$cc) 2315 // Emits: (MOVCCi:i32 GPR:i32:$false, 2316 // (so_imm:i32 (imm:i32):$true), (imm:i32):$cc) 2317 // Pattern complexity = 10 cost = 1 size = 0 2318 if (Subtarget->isThumb()) { 2319 SDNode *Res = SelectT2CMOVImmOp(N, FalseVal, TrueVal, 2320 CCVal, CCR, InFlag); 2321 if (!Res) 2322 Res = SelectT2CMOVImmOp(N, TrueVal, FalseVal, 2323 ARMCC::getOppositeCondition(CCVal), CCR, InFlag); 2324 if (Res) 2325 return Res; 2326 } else { 2327 SDNode *Res = SelectARMCMOVImmOp(N, FalseVal, TrueVal, 2328 CCVal, CCR, InFlag); 2329 if (!Res) 2330 Res = SelectARMCMOVImmOp(N, TrueVal, FalseVal, 2331 ARMCC::getOppositeCondition(CCVal), CCR, InFlag); 2332 if (Res) 2333 return Res; 2334 } 2335 } 2336 2337 // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) 2338 // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) 2339 // Pattern complexity = 6 cost = 1 size = 0 2340 // 2341 // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) 2342 // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) 2343 // Pattern complexity = 6 cost = 11 size = 0 2344 // 2345 // Also VMOVScc and VMOVDcc. 2346 SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32); 2347 SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag }; 2348 unsigned Opc = 0; 2349 switch (VT.getSimpleVT().SimpleTy) { 2350 default: llvm_unreachable("Illegal conditional move type!"); 2351 case MVT::i32: 2352 Opc = Subtarget->isThumb() 2353 ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo) 2354 : ARM::MOVCCr; 2355 break; 2356 case MVT::f32: 2357 Opc = ARM::VMOVScc; 2358 break; 2359 case MVT::f64: 2360 Opc = ARM::VMOVDcc; 2361 break; 2362 } 2363 return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5); 2364 } 2365 2366 SDNode *ARMDAGToDAGISel::SelectConditionalOp(SDNode *N) { 2367 SDValue FalseVal = N->getOperand(0); 2368 SDValue TrueVal = N->getOperand(1); 2369 ARMCC::CondCodes CCVal = 2370 (ARMCC::CondCodes)cast<ConstantSDNode>(N->getOperand(2))->getZExtValue(); 2371 SDValue CCR = N->getOperand(3); 2372 assert(CCR.getOpcode() == ISD::Register); 2373 SDValue InFlag = N->getOperand(4); 2374 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); 2375 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2376 2377 if (Subtarget->isThumb()) { 2378 SDValue CPTmp0; 2379 SDValue CPTmp1; 2380 if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) { 2381 unsigned Opc; 2382 switch (N->getOpcode()) { 2383 default: llvm_unreachable("Unexpected node"); 2384 case ARMISD::CAND: Opc = ARM::t2ANDCCrs; break; 2385 case ARMISD::COR: Opc = ARM::t2ORRCCrs; break; 2386 case ARMISD::CXOR: Opc = ARM::t2EORCCrs; break; 2387 } 2388 SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CC, CCR, Reg0, InFlag }; 2389 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 7); 2390 } 2391 2392 ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal); 2393 if (T) { 2394 unsigned TrueImm = T->getZExtValue(); 2395 if (is_t2_so_imm(TrueImm)) { 2396 unsigned Opc; 2397 switch (N->getOpcode()) { 2398 default: llvm_unreachable("Unexpected node"); 2399 case ARMISD::CAND: Opc = ARM::t2ANDCCri; break; 2400 case ARMISD::COR: Opc = ARM::t2ORRCCri; break; 2401 case ARMISD::CXOR: Opc = ARM::t2EORCCri; break; 2402 } 2403 SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32); 2404 SDValue Ops[] = { FalseVal, True, CC, CCR, Reg0, InFlag }; 2405 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 6); 2406 } 2407 } 2408 2409 unsigned Opc; 2410 switch (N->getOpcode()) { 2411 default: llvm_unreachable("Unexpected node"); 2412 case ARMISD::CAND: Opc = ARM::t2ANDCCrr; break; 2413 case ARMISD::COR: Opc = ARM::t2ORRCCrr; break; 2414 case ARMISD::CXOR: Opc = ARM::t2EORCCrr; break; 2415 } 2416 SDValue Ops[] = { FalseVal, TrueVal, CC, CCR, Reg0, InFlag }; 2417 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 6); 2418 } 2419 2420 SDValue CPTmp0; 2421 SDValue CPTmp1; 2422 SDValue CPTmp2; 2423 if (SelectImmShifterOperand(TrueVal, CPTmp0, CPTmp2)) { 2424 unsigned Opc; 2425 switch (N->getOpcode()) { 2426 default: llvm_unreachable("Unexpected node"); 2427 case ARMISD::CAND: Opc = ARM::ANDCCrsi; break; 2428 case ARMISD::COR: Opc = ARM::ORRCCrsi; break; 2429 case ARMISD::CXOR: Opc = ARM::EORCCrsi; break; 2430 } 2431 SDValue Ops[] = { FalseVal, CPTmp0, CPTmp2, CC, CCR, Reg0, InFlag }; 2432 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 7); 2433 } 2434 2435 if (SelectRegShifterOperand(TrueVal, CPTmp0, CPTmp1, CPTmp2)) { 2436 unsigned Opc; 2437 switch (N->getOpcode()) { 2438 default: llvm_unreachable("Unexpected node"); 2439 case ARMISD::CAND: Opc = ARM::ANDCCrsr; break; 2440 case ARMISD::COR: Opc = ARM::ORRCCrsr; break; 2441 case ARMISD::CXOR: Opc = ARM::EORCCrsr; break; 2442 } 2443 SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, Reg0, InFlag }; 2444 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 8); 2445 } 2446 2447 ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal); 2448 if (T) { 2449 unsigned TrueImm = T->getZExtValue(); 2450 if (is_so_imm(TrueImm)) { 2451 unsigned Opc; 2452 switch (N->getOpcode()) { 2453 default: llvm_unreachable("Unexpected node"); 2454 case ARMISD::CAND: Opc = ARM::ANDCCri; break; 2455 case ARMISD::COR: Opc = ARM::ORRCCri; break; 2456 case ARMISD::CXOR: Opc = ARM::EORCCri; break; 2457 } 2458 SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32); 2459 SDValue Ops[] = { FalseVal, True, CC, CCR, Reg0, InFlag }; 2460 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 6); 2461 } 2462 } 2463 2464 unsigned Opc; 2465 switch (N->getOpcode()) { 2466 default: llvm_unreachable("Unexpected node"); 2467 case ARMISD::CAND: Opc = ARM::ANDCCrr; break; 2468 case ARMISD::COR: Opc = ARM::ORRCCrr; break; 2469 case ARMISD::CXOR: Opc = ARM::EORCCrr; break; 2470 } 2471 SDValue Ops[] = { FalseVal, TrueVal, CC, CCR, Reg0, InFlag }; 2472 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 6); 2473 } 2474 2475 /// Target-specific DAG combining for ISD::XOR. 2476 /// Target-independent combining lowers SELECT_CC nodes of the form 2477 /// select_cc setg[ge] X, 0, X, -X 2478 /// select_cc setgt X, -1, X, -X 2479 /// select_cc setl[te] X, 0, -X, X 2480 /// select_cc setlt X, 1, -X, X 2481 /// which represent Integer ABS into: 2482 /// Y = sra (X, size(X)-1); xor (add (X, Y), Y) 2483 /// ARM instruction selection detects the latter and matches it to 2484 /// ARM::ABS or ARM::t2ABS machine node. 2485 SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){ 2486 SDValue XORSrc0 = N->getOperand(0); 2487 SDValue XORSrc1 = N->getOperand(1); 2488 EVT VT = N->getValueType(0); 2489 2490 if (Subtarget->isThumb1Only()) 2491 return NULL; 2492 2493 if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA) 2494 return NULL; 2495 2496 SDValue ADDSrc0 = XORSrc0.getOperand(0); 2497 SDValue ADDSrc1 = XORSrc0.getOperand(1); 2498 SDValue SRASrc0 = XORSrc1.getOperand(0); 2499 SDValue SRASrc1 = XORSrc1.getOperand(1); 2500 ConstantSDNode *SRAConstant = dyn_cast<ConstantSDNode>(SRASrc1); 2501 EVT XType = SRASrc0.getValueType(); 2502 unsigned Size = XType.getSizeInBits() - 1; 2503 2504 if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 && 2505 XType.isInteger() && SRAConstant != NULL && 2506 Size == SRAConstant->getZExtValue()) { 2507 unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS; 2508 return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0); 2509 } 2510 2511 return NULL; 2512 } 2513 2514 SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) { 2515 // The only time a CONCAT_VECTORS operation can have legal types is when 2516 // two 64-bit vectors are concatenated to a 128-bit vector. 2517 EVT VT = N->getValueType(0); 2518 if (!VT.is128BitVector() || N->getNumOperands() != 2) 2519 llvm_unreachable("unexpected CONCAT_VECTORS"); 2520 return PairDRegs(VT, N->getOperand(0), N->getOperand(1)); 2521 } 2522 2523 SDNode *ARMDAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) { 2524 SmallVector<SDValue, 6> Ops; 2525 Ops.push_back(Node->getOperand(1)); // Ptr 2526 Ops.push_back(Node->getOperand(2)); // Low part of Val1 2527 Ops.push_back(Node->getOperand(3)); // High part of Val1 2528 if (Opc == ARM::ATOMCMPXCHG6432) { 2529 Ops.push_back(Node->getOperand(4)); // Low part of Val2 2530 Ops.push_back(Node->getOperand(5)); // High part of Val2 2531 } 2532 Ops.push_back(Node->getOperand(0)); // Chain 2533 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 2534 MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 2535 SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(), 2536 MVT::i32, MVT::i32, MVT::Other, 2537 Ops.data() ,Ops.size()); 2538 cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp + 1); 2539 return ResNode; 2540 } 2541 2542 SDNode *ARMDAGToDAGISel::Select(SDNode *N) { 2543 DebugLoc dl = N->getDebugLoc(); 2544 2545 if (N->isMachineOpcode()) 2546 return NULL; // Already selected. 2547 2548 switch (N->getOpcode()) { 2549 default: break; 2550 case ISD::XOR: { 2551 // Select special operations if XOR node forms integer ABS pattern 2552 SDNode *ResNode = SelectABSOp(N); 2553 if (ResNode) 2554 return ResNode; 2555 // Other cases are autogenerated. 2556 break; 2557 } 2558 case ISD::Constant: { 2559 unsigned Val = cast<ConstantSDNode>(N)->getZExtValue(); 2560 bool UseCP = true; 2561 if (Subtarget->hasThumb2()) 2562 // Thumb2-aware targets have the MOVT instruction, so all immediates can 2563 // be done with MOV + MOVT, at worst. 2564 UseCP = 0; 2565 else { 2566 if (Subtarget->isThumb()) { 2567 UseCP = (Val > 255 && // MOV 2568 ~Val > 255 && // MOV + MVN 2569 !ARM_AM::isThumbImmShiftedVal(Val)); // MOV + LSL 2570 } else 2571 UseCP = (ARM_AM::getSOImmVal(Val) == -1 && // MOV 2572 ARM_AM::getSOImmVal(~Val) == -1 && // MVN 2573 !ARM_AM::isSOImmTwoPartVal(Val)); // two instrs. 2574 } 2575 2576 if (UseCP) { 2577 SDValue CPIdx = 2578 CurDAG->getTargetConstantPool(ConstantInt::get( 2579 Type::getInt32Ty(*CurDAG->getContext()), Val), 2580 TLI.getPointerTy()); 2581 2582 SDNode *ResNode; 2583 if (Subtarget->isThumb1Only()) { 2584 SDValue Pred = getAL(CurDAG); 2585 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2586 SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() }; 2587 ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other, 2588 Ops, 4); 2589 } else { 2590 SDValue Ops[] = { 2591 CPIdx, 2592 CurDAG->getTargetConstant(0, MVT::i32), 2593 getAL(CurDAG), 2594 CurDAG->getRegister(0, MVT::i32), 2595 CurDAG->getEntryNode() 2596 }; 2597 ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other, 2598 Ops, 5); 2599 } 2600 ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0)); 2601 return NULL; 2602 } 2603 2604 // Other cases are autogenerated. 2605 break; 2606 } 2607 case ISD::FrameIndex: { 2608 // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm. 2609 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 2610 SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 2611 if (Subtarget->isThumb1Only()) { 2612 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32), 2613 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2614 return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, Ops, 4); 2615 } else { 2616 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ? 2617 ARM::t2ADDri : ARM::ADDri); 2618 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32), 2619 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2620 CurDAG->getRegister(0, MVT::i32) }; 2621 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); 2622 } 2623 } 2624 case ISD::SRL: 2625 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false)) 2626 return I; 2627 break; 2628 case ISD::SRA: 2629 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true)) 2630 return I; 2631 break; 2632 case ISD::MUL: 2633 if (Subtarget->isThumb1Only()) 2634 break; 2635 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 2636 unsigned RHSV = C->getZExtValue(); 2637 if (!RHSV) break; 2638 if (isPowerOf2_32(RHSV-1)) { // 2^n+1? 2639 unsigned ShImm = Log2_32(RHSV-1); 2640 if (ShImm >= 32) 2641 break; 2642 SDValue V = N->getOperand(0); 2643 ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); 2644 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32); 2645 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2646 if (Subtarget->isThumb()) { 2647 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2648 return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6); 2649 } else { 2650 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2651 return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops, 7); 2652 } 2653 } 2654 if (isPowerOf2_32(RHSV+1)) { // 2^n-1? 2655 unsigned ShImm = Log2_32(RHSV+1); 2656 if (ShImm >= 32) 2657 break; 2658 SDValue V = N->getOperand(0); 2659 ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); 2660 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32); 2661 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2662 if (Subtarget->isThumb()) { 2663 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2664 return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 6); 2665 } else { 2666 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2667 return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops, 7); 2668 } 2669 } 2670 } 2671 break; 2672 case ISD::AND: { 2673 // Check for unsigned bitfield extract 2674 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false)) 2675 return I; 2676 2677 // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits 2678 // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits 2679 // are entirely contributed by c2 and lower 16-bits are entirely contributed 2680 // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)). 2681 // Select it to: "movt x, ((c1 & 0xffff) >> 16) 2682 EVT VT = N->getValueType(0); 2683 if (VT != MVT::i32) 2684 break; 2685 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2()) 2686 ? ARM::t2MOVTi16 2687 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0); 2688 if (!Opc) 2689 break; 2690 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); 2691 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 2692 if (!N1C) 2693 break; 2694 if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) { 2695 SDValue N2 = N0.getOperand(1); 2696 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); 2697 if (!N2C) 2698 break; 2699 unsigned N1CVal = N1C->getZExtValue(); 2700 unsigned N2CVal = N2C->getZExtValue(); 2701 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) && 2702 (N1CVal & 0xffffU) == 0xffffU && 2703 (N2CVal & 0xffffU) == 0x0U) { 2704 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16, 2705 MVT::i32); 2706 SDValue Ops[] = { N0.getOperand(0), Imm16, 2707 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2708 return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4); 2709 } 2710 } 2711 break; 2712 } 2713 case ARMISD::VMOVRRD: 2714 return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32, 2715 N->getOperand(0), getAL(CurDAG), 2716 CurDAG->getRegister(0, MVT::i32)); 2717 case ISD::UMUL_LOHI: { 2718 if (Subtarget->isThumb1Only()) 2719 break; 2720 if (Subtarget->isThumb()) { 2721 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2722 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2723 CurDAG->getRegister(0, MVT::i32) }; 2724 return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32,Ops,4); 2725 } else { 2726 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2727 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2728 CurDAG->getRegister(0, MVT::i32) }; 2729 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2730 ARM::UMULL : ARM::UMULLv5, 2731 dl, MVT::i32, MVT::i32, Ops, 5); 2732 } 2733 } 2734 case ISD::SMUL_LOHI: { 2735 if (Subtarget->isThumb1Only()) 2736 break; 2737 if (Subtarget->isThumb()) { 2738 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2739 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2740 return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32,Ops,4); 2741 } else { 2742 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2743 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2744 CurDAG->getRegister(0, MVT::i32) }; 2745 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2746 ARM::SMULL : ARM::SMULLv5, 2747 dl, MVT::i32, MVT::i32, Ops, 5); 2748 } 2749 } 2750 case ISD::LOAD: { 2751 SDNode *ResNode = 0; 2752 if (Subtarget->isThumb() && Subtarget->hasThumb2()) 2753 ResNode = SelectT2IndexedLoad(N); 2754 else 2755 ResNode = SelectARMIndexedLoad(N); 2756 if (ResNode) 2757 return ResNode; 2758 // Other cases are autogenerated. 2759 break; 2760 } 2761 case ARMISD::BRCOND: { 2762 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2763 // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc) 2764 // Pattern complexity = 6 cost = 1 size = 0 2765 2766 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2767 // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc) 2768 // Pattern complexity = 6 cost = 1 size = 0 2769 2770 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2771 // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc) 2772 // Pattern complexity = 6 cost = 1 size = 0 2773 2774 unsigned Opc = Subtarget->isThumb() ? 2775 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc; 2776 SDValue Chain = N->getOperand(0); 2777 SDValue N1 = N->getOperand(1); 2778 SDValue N2 = N->getOperand(2); 2779 SDValue N3 = N->getOperand(3); 2780 SDValue InFlag = N->getOperand(4); 2781 assert(N1.getOpcode() == ISD::BasicBlock); 2782 assert(N2.getOpcode() == ISD::Constant); 2783 assert(N3.getOpcode() == ISD::Register); 2784 2785 SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) 2786 cast<ConstantSDNode>(N2)->getZExtValue()), 2787 MVT::i32); 2788 SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag }; 2789 SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, 2790 MVT::Glue, Ops, 5); 2791 Chain = SDValue(ResNode, 0); 2792 if (N->getNumValues() == 2) { 2793 InFlag = SDValue(ResNode, 1); 2794 ReplaceUses(SDValue(N, 1), InFlag); 2795 } 2796 ReplaceUses(SDValue(N, 0), 2797 SDValue(Chain.getNode(), Chain.getResNo())); 2798 return NULL; 2799 } 2800 case ARMISD::CMOV: 2801 return SelectCMOVOp(N); 2802 case ARMISD::CAND: 2803 case ARMISD::COR: 2804 case ARMISD::CXOR: 2805 return SelectConditionalOp(N); 2806 case ARMISD::VZIP: { 2807 unsigned Opc = 0; 2808 EVT VT = N->getValueType(0); 2809 switch (VT.getSimpleVT().SimpleTy) { 2810 default: return NULL; 2811 case MVT::v8i8: Opc = ARM::VZIPd8; break; 2812 case MVT::v4i16: Opc = ARM::VZIPd16; break; 2813 case MVT::v2f32: 2814 // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. 2815 case MVT::v2i32: Opc = ARM::VTRNd32; break; 2816 case MVT::v16i8: Opc = ARM::VZIPq8; break; 2817 case MVT::v8i16: Opc = ARM::VZIPq16; break; 2818 case MVT::v4f32: 2819 case MVT::v4i32: Opc = ARM::VZIPq32; break; 2820 } 2821 SDValue Pred = getAL(CurDAG); 2822 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2823 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2824 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4); 2825 } 2826 case ARMISD::VUZP: { 2827 unsigned Opc = 0; 2828 EVT VT = N->getValueType(0); 2829 switch (VT.getSimpleVT().SimpleTy) { 2830 default: return NULL; 2831 case MVT::v8i8: Opc = ARM::VUZPd8; break; 2832 case MVT::v4i16: Opc = ARM::VUZPd16; break; 2833 case MVT::v2f32: 2834 // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. 2835 case MVT::v2i32: Opc = ARM::VTRNd32; break; 2836 case MVT::v16i8: Opc = ARM::VUZPq8; break; 2837 case MVT::v8i16: Opc = ARM::VUZPq16; break; 2838 case MVT::v4f32: 2839 case MVT::v4i32: Opc = ARM::VUZPq32; break; 2840 } 2841 SDValue Pred = getAL(CurDAG); 2842 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2843 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2844 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4); 2845 } 2846 case ARMISD::VTRN: { 2847 unsigned Opc = 0; 2848 EVT VT = N->getValueType(0); 2849 switch (VT.getSimpleVT().SimpleTy) { 2850 default: return NULL; 2851 case MVT::v8i8: Opc = ARM::VTRNd8; break; 2852 case MVT::v4i16: Opc = ARM::VTRNd16; break; 2853 case MVT::v2f32: 2854 case MVT::v2i32: Opc = ARM::VTRNd32; break; 2855 case MVT::v16i8: Opc = ARM::VTRNq8; break; 2856 case MVT::v8i16: Opc = ARM::VTRNq16; break; 2857 case MVT::v4f32: 2858 case MVT::v4i32: Opc = ARM::VTRNq32; break; 2859 } 2860 SDValue Pred = getAL(CurDAG); 2861 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2862 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2863 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4); 2864 } 2865 case ARMISD::BUILD_VECTOR: { 2866 EVT VecVT = N->getValueType(0); 2867 EVT EltVT = VecVT.getVectorElementType(); 2868 unsigned NumElts = VecVT.getVectorNumElements(); 2869 if (EltVT == MVT::f64) { 2870 assert(NumElts == 2 && "unexpected type for BUILD_VECTOR"); 2871 return PairDRegs(VecVT, N->getOperand(0), N->getOperand(1)); 2872 } 2873 assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR"); 2874 if (NumElts == 2) 2875 return PairSRegs(VecVT, N->getOperand(0), N->getOperand(1)); 2876 assert(NumElts == 4 && "unexpected type for BUILD_VECTOR"); 2877 return QuadSRegs(VecVT, N->getOperand(0), N->getOperand(1), 2878 N->getOperand(2), N->getOperand(3)); 2879 } 2880 2881 case ARMISD::VLD2DUP: { 2882 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16, 2883 ARM::VLD2DUPd32 }; 2884 return SelectVLDDup(N, false, 2, Opcodes); 2885 } 2886 2887 case ARMISD::VLD3DUP: { 2888 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo, 2889 ARM::VLD3DUPd16Pseudo, 2890 ARM::VLD3DUPd32Pseudo }; 2891 return SelectVLDDup(N, false, 3, Opcodes); 2892 } 2893 2894 case ARMISD::VLD4DUP: { 2895 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo, 2896 ARM::VLD4DUPd16Pseudo, 2897 ARM::VLD4DUPd32Pseudo }; 2898 return SelectVLDDup(N, false, 4, Opcodes); 2899 } 2900 2901 case ARMISD::VLD2DUP_UPD: { 2902 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed, 2903 ARM::VLD2DUPd16wb_fixed, 2904 ARM::VLD2DUPd32wb_fixed }; 2905 return SelectVLDDup(N, true, 2, Opcodes); 2906 } 2907 2908 case ARMISD::VLD3DUP_UPD: { 2909 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD, 2910 ARM::VLD3DUPd16Pseudo_UPD, 2911 ARM::VLD3DUPd32Pseudo_UPD }; 2912 return SelectVLDDup(N, true, 3, Opcodes); 2913 } 2914 2915 case ARMISD::VLD4DUP_UPD: { 2916 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD, 2917 ARM::VLD4DUPd16Pseudo_UPD, 2918 ARM::VLD4DUPd32Pseudo_UPD }; 2919 return SelectVLDDup(N, true, 4, Opcodes); 2920 } 2921 2922 case ARMISD::VLD1_UPD: { 2923 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed, 2924 ARM::VLD1d16wb_fixed, 2925 ARM::VLD1d32wb_fixed, 2926 ARM::VLD1d64wb_fixed }; 2927 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed, 2928 ARM::VLD1q16wb_fixed, 2929 ARM::VLD1q32wb_fixed, 2930 ARM::VLD1q64wb_fixed }; 2931 return SelectVLD(N, true, 1, DOpcodes, QOpcodes, 0); 2932 } 2933 2934 case ARMISD::VLD2_UPD: { 2935 static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed, 2936 ARM::VLD2d16wb_fixed, 2937 ARM::VLD2d32wb_fixed, 2938 ARM::VLD1q64wb_fixed}; 2939 static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed, 2940 ARM::VLD2q16PseudoWB_fixed, 2941 ARM::VLD2q32PseudoWB_fixed }; 2942 return SelectVLD(N, true, 2, DOpcodes, QOpcodes, 0); 2943 } 2944 2945 case ARMISD::VLD3_UPD: { 2946 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD, 2947 ARM::VLD3d16Pseudo_UPD, 2948 ARM::VLD3d32Pseudo_UPD, 2949 ARM::VLD1q64wb_fixed}; 2950 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD, 2951 ARM::VLD3q16Pseudo_UPD, 2952 ARM::VLD3q32Pseudo_UPD }; 2953 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD, 2954 ARM::VLD3q16oddPseudo_UPD, 2955 ARM::VLD3q32oddPseudo_UPD }; 2956 return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); 2957 } 2958 2959 case ARMISD::VLD4_UPD: { 2960 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD, 2961 ARM::VLD4d16Pseudo_UPD, 2962 ARM::VLD4d32Pseudo_UPD, 2963 ARM::VLD1q64wb_fixed}; 2964 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD, 2965 ARM::VLD4q16Pseudo_UPD, 2966 ARM::VLD4q32Pseudo_UPD }; 2967 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD, 2968 ARM::VLD4q16oddPseudo_UPD, 2969 ARM::VLD4q32oddPseudo_UPD }; 2970 return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); 2971 } 2972 2973 case ARMISD::VLD2LN_UPD: { 2974 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD, 2975 ARM::VLD2LNd16Pseudo_UPD, 2976 ARM::VLD2LNd32Pseudo_UPD }; 2977 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD, 2978 ARM::VLD2LNq32Pseudo_UPD }; 2979 return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes); 2980 } 2981 2982 case ARMISD::VLD3LN_UPD: { 2983 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD, 2984 ARM::VLD3LNd16Pseudo_UPD, 2985 ARM::VLD3LNd32Pseudo_UPD }; 2986 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD, 2987 ARM::VLD3LNq32Pseudo_UPD }; 2988 return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes); 2989 } 2990 2991 case ARMISD::VLD4LN_UPD: { 2992 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD, 2993 ARM::VLD4LNd16Pseudo_UPD, 2994 ARM::VLD4LNd32Pseudo_UPD }; 2995 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD, 2996 ARM::VLD4LNq32Pseudo_UPD }; 2997 return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes); 2998 } 2999 3000 case ARMISD::VST1_UPD: { 3001 static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed, 3002 ARM::VST1d16wb_fixed, 3003 ARM::VST1d32wb_fixed, 3004 ARM::VST1d64wb_fixed }; 3005 static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed, 3006 ARM::VST1q16wb_fixed, 3007 ARM::VST1q32wb_fixed, 3008 ARM::VST1q64wb_fixed }; 3009 return SelectVST(N, true, 1, DOpcodes, QOpcodes, 0); 3010 } 3011 3012 case ARMISD::VST2_UPD: { 3013 static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed, 3014 ARM::VST2d16wb_fixed, 3015 ARM::VST2d32wb_fixed, 3016 ARM::VST1q64wb_fixed}; 3017 static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed, 3018 ARM::VST2q16PseudoWB_fixed, 3019 ARM::VST2q32PseudoWB_fixed }; 3020 return SelectVST(N, true, 2, DOpcodes, QOpcodes, 0); 3021 } 3022 3023 case ARMISD::VST3_UPD: { 3024 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD, 3025 ARM::VST3d16Pseudo_UPD, 3026 ARM::VST3d32Pseudo_UPD, 3027 ARM::VST1d64TPseudoWB_fixed}; 3028 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD, 3029 ARM::VST3q16Pseudo_UPD, 3030 ARM::VST3q32Pseudo_UPD }; 3031 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD, 3032 ARM::VST3q16oddPseudo_UPD, 3033 ARM::VST3q32oddPseudo_UPD }; 3034 return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); 3035 } 3036 3037 case ARMISD::VST4_UPD: { 3038 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD, 3039 ARM::VST4d16Pseudo_UPD, 3040 ARM::VST4d32Pseudo_UPD, 3041 ARM::VST1d64QPseudoWB_fixed}; 3042 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD, 3043 ARM::VST4q16Pseudo_UPD, 3044 ARM::VST4q32Pseudo_UPD }; 3045 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD, 3046 ARM::VST4q16oddPseudo_UPD, 3047 ARM::VST4q32oddPseudo_UPD }; 3048 return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); 3049 } 3050 3051 case ARMISD::VST2LN_UPD: { 3052 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD, 3053 ARM::VST2LNd16Pseudo_UPD, 3054 ARM::VST2LNd32Pseudo_UPD }; 3055 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD, 3056 ARM::VST2LNq32Pseudo_UPD }; 3057 return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes); 3058 } 3059 3060 case ARMISD::VST3LN_UPD: { 3061 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD, 3062 ARM::VST3LNd16Pseudo_UPD, 3063 ARM::VST3LNd32Pseudo_UPD }; 3064 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD, 3065 ARM::VST3LNq32Pseudo_UPD }; 3066 return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes); 3067 } 3068 3069 case ARMISD::VST4LN_UPD: { 3070 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD, 3071 ARM::VST4LNd16Pseudo_UPD, 3072 ARM::VST4LNd32Pseudo_UPD }; 3073 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD, 3074 ARM::VST4LNq32Pseudo_UPD }; 3075 return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes); 3076 } 3077 3078 case ISD::INTRINSIC_VOID: 3079 case ISD::INTRINSIC_W_CHAIN: { 3080 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 3081 switch (IntNo) { 3082 default: 3083 break; 3084 3085 case Intrinsic::arm_ldrexd: { 3086 SDValue MemAddr = N->getOperand(2); 3087 DebugLoc dl = N->getDebugLoc(); 3088 SDValue Chain = N->getOperand(0); 3089 3090 unsigned NewOpc = ARM::LDREXD; 3091 if (Subtarget->isThumb() && Subtarget->hasThumb2()) 3092 NewOpc = ARM::t2LDREXD; 3093 3094 // arm_ldrexd returns a i64 value in {i32, i32} 3095 std::vector<EVT> ResTys; 3096 ResTys.push_back(MVT::i32); 3097 ResTys.push_back(MVT::i32); 3098 ResTys.push_back(MVT::Other); 3099 3100 // place arguments in the right order 3101 SmallVector<SDValue, 7> Ops; 3102 Ops.push_back(MemAddr); 3103 Ops.push_back(getAL(CurDAG)); 3104 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); 3105 Ops.push_back(Chain); 3106 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(), 3107 Ops.size()); 3108 // Transfer memoperands. 3109 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 3110 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 3111 cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1); 3112 3113 // Until there's support for specifing explicit register constraints 3114 // like the use of even/odd register pair, hardcode ldrexd to always 3115 // use the pair [R0, R1] to hold the load result. 3116 Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ARM::R0, 3117 SDValue(Ld, 0), SDValue(0,0)); 3118 Chain = CurDAG->getCopyToReg(Chain, dl, ARM::R1, 3119 SDValue(Ld, 1), Chain.getValue(1)); 3120 3121 // Remap uses. 3122 SDValue Glue = Chain.getValue(1); 3123 if (!SDValue(N, 0).use_empty()) { 3124 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 3125 ARM::R0, MVT::i32, Glue); 3126 Glue = Result.getValue(2); 3127 ReplaceUses(SDValue(N, 0), Result); 3128 } 3129 if (!SDValue(N, 1).use_empty()) { 3130 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 3131 ARM::R1, MVT::i32, Glue); 3132 Glue = Result.getValue(2); 3133 ReplaceUses(SDValue(N, 1), Result); 3134 } 3135 3136 ReplaceUses(SDValue(N, 2), SDValue(Ld, 2)); 3137 return NULL; 3138 } 3139 3140 case Intrinsic::arm_strexd: { 3141 DebugLoc dl = N->getDebugLoc(); 3142 SDValue Chain = N->getOperand(0); 3143 SDValue Val0 = N->getOperand(2); 3144 SDValue Val1 = N->getOperand(3); 3145 SDValue MemAddr = N->getOperand(4); 3146 3147 // Until there's support for specifing explicit register constraints 3148 // like the use of even/odd register pair, hardcode strexd to always 3149 // use the pair [R2, R3] to hold the i64 (i32, i32) value to be stored. 3150 Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ARM::R2, Val0, 3151 SDValue(0, 0)); 3152 Chain = CurDAG->getCopyToReg(Chain, dl, ARM::R3, Val1, Chain.getValue(1)); 3153 3154 SDValue Glue = Chain.getValue(1); 3155 Val0 = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 3156 ARM::R2, MVT::i32, Glue); 3157 Glue = Val0.getValue(1); 3158 Val1 = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 3159 ARM::R3, MVT::i32, Glue); 3160 3161 // Store exclusive double return a i32 value which is the return status 3162 // of the issued store. 3163 std::vector<EVT> ResTys; 3164 ResTys.push_back(MVT::i32); 3165 ResTys.push_back(MVT::Other); 3166 3167 // place arguments in the right order 3168 SmallVector<SDValue, 7> Ops; 3169 Ops.push_back(Val0); 3170 Ops.push_back(Val1); 3171 Ops.push_back(MemAddr); 3172 Ops.push_back(getAL(CurDAG)); 3173 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); 3174 Ops.push_back(Chain); 3175 3176 unsigned NewOpc = ARM::STREXD; 3177 if (Subtarget->isThumb() && Subtarget->hasThumb2()) 3178 NewOpc = ARM::t2STREXD; 3179 3180 SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(), 3181 Ops.size()); 3182 // Transfer memoperands. 3183 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 3184 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 3185 cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1); 3186 3187 return St; 3188 } 3189 3190 case Intrinsic::arm_neon_vld1: { 3191 static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16, 3192 ARM::VLD1d32, ARM::VLD1d64 }; 3193 static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16, 3194 ARM::VLD1q32, ARM::VLD1q64}; 3195 return SelectVLD(N, false, 1, DOpcodes, QOpcodes, 0); 3196 } 3197 3198 case Intrinsic::arm_neon_vld2: { 3199 static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16, 3200 ARM::VLD2d32, ARM::VLD1q64 }; 3201 static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo, 3202 ARM::VLD2q32Pseudo }; 3203 return SelectVLD(N, false, 2, DOpcodes, QOpcodes, 0); 3204 } 3205 3206 case Intrinsic::arm_neon_vld3: { 3207 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo, 3208 ARM::VLD3d16Pseudo, 3209 ARM::VLD3d32Pseudo, 3210 ARM::VLD1d64TPseudo }; 3211 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD, 3212 ARM::VLD3q16Pseudo_UPD, 3213 ARM::VLD3q32Pseudo_UPD }; 3214 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo, 3215 ARM::VLD3q16oddPseudo, 3216 ARM::VLD3q32oddPseudo }; 3217 return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); 3218 } 3219 3220 case Intrinsic::arm_neon_vld4: { 3221 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo, 3222 ARM::VLD4d16Pseudo, 3223 ARM::VLD4d32Pseudo, 3224 ARM::VLD1d64QPseudo }; 3225 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD, 3226 ARM::VLD4q16Pseudo_UPD, 3227 ARM::VLD4q32Pseudo_UPD }; 3228 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo, 3229 ARM::VLD4q16oddPseudo, 3230 ARM::VLD4q32oddPseudo }; 3231 return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); 3232 } 3233 3234 case Intrinsic::arm_neon_vld2lane: { 3235 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo, 3236 ARM::VLD2LNd16Pseudo, 3237 ARM::VLD2LNd32Pseudo }; 3238 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo, 3239 ARM::VLD2LNq32Pseudo }; 3240 return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes); 3241 } 3242 3243 case Intrinsic::arm_neon_vld3lane: { 3244 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo, 3245 ARM::VLD3LNd16Pseudo, 3246 ARM::VLD3LNd32Pseudo }; 3247 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo, 3248 ARM::VLD3LNq32Pseudo }; 3249 return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes); 3250 } 3251 3252 case Intrinsic::arm_neon_vld4lane: { 3253 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo, 3254 ARM::VLD4LNd16Pseudo, 3255 ARM::VLD4LNd32Pseudo }; 3256 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo, 3257 ARM::VLD4LNq32Pseudo }; 3258 return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes); 3259 } 3260 3261 case Intrinsic::arm_neon_vst1: { 3262 static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16, 3263 ARM::VST1d32, ARM::VST1d64 }; 3264 static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16, 3265 ARM::VST1q32, ARM::VST1q64 }; 3266 return SelectVST(N, false, 1, DOpcodes, QOpcodes, 0); 3267 } 3268 3269 case Intrinsic::arm_neon_vst2: { 3270 static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16, 3271 ARM::VST2d32, ARM::VST1q64 }; 3272 static uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo, 3273 ARM::VST2q32Pseudo }; 3274 return SelectVST(N, false, 2, DOpcodes, QOpcodes, 0); 3275 } 3276 3277 case Intrinsic::arm_neon_vst3: { 3278 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo, 3279 ARM::VST3d16Pseudo, 3280 ARM::VST3d32Pseudo, 3281 ARM::VST1d64TPseudo }; 3282 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD, 3283 ARM::VST3q16Pseudo_UPD, 3284 ARM::VST3q32Pseudo_UPD }; 3285 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo, 3286 ARM::VST3q16oddPseudo, 3287 ARM::VST3q32oddPseudo }; 3288 return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); 3289 } 3290 3291 case Intrinsic::arm_neon_vst4: { 3292 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo, 3293 ARM::VST4d16Pseudo, 3294 ARM::VST4d32Pseudo, 3295 ARM::VST1d64QPseudo }; 3296 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD, 3297 ARM::VST4q16Pseudo_UPD, 3298 ARM::VST4q32Pseudo_UPD }; 3299 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo, 3300 ARM::VST4q16oddPseudo, 3301 ARM::VST4q32oddPseudo }; 3302 return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); 3303 } 3304 3305 case Intrinsic::arm_neon_vst2lane: { 3306 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo, 3307 ARM::VST2LNd16Pseudo, 3308 ARM::VST2LNd32Pseudo }; 3309 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo, 3310 ARM::VST2LNq32Pseudo }; 3311 return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes); 3312 } 3313 3314 case Intrinsic::arm_neon_vst3lane: { 3315 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo, 3316 ARM::VST3LNd16Pseudo, 3317 ARM::VST3LNd32Pseudo }; 3318 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo, 3319 ARM::VST3LNq32Pseudo }; 3320 return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes); 3321 } 3322 3323 case Intrinsic::arm_neon_vst4lane: { 3324 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo, 3325 ARM::VST4LNd16Pseudo, 3326 ARM::VST4LNd32Pseudo }; 3327 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo, 3328 ARM::VST4LNq32Pseudo }; 3329 return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes); 3330 } 3331 } 3332 break; 3333 } 3334 3335 case ISD::INTRINSIC_WO_CHAIN: { 3336 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 3337 switch (IntNo) { 3338 default: 3339 break; 3340 3341 case Intrinsic::arm_neon_vtbl2: 3342 return SelectVTBL(N, false, 2, ARM::VTBL2); 3343 case Intrinsic::arm_neon_vtbl3: 3344 return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo); 3345 case Intrinsic::arm_neon_vtbl4: 3346 return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo); 3347 3348 case Intrinsic::arm_neon_vtbx2: 3349 return SelectVTBL(N, true, 2, ARM::VTBX2); 3350 case Intrinsic::arm_neon_vtbx3: 3351 return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo); 3352 case Intrinsic::arm_neon_vtbx4: 3353 return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo); 3354 } 3355 break; 3356 } 3357 3358 case ARMISD::VTBL1: { 3359 DebugLoc dl = N->getDebugLoc(); 3360 EVT VT = N->getValueType(0); 3361 SmallVector<SDValue, 6> Ops; 3362 3363 Ops.push_back(N->getOperand(0)); 3364 Ops.push_back(N->getOperand(1)); 3365 Ops.push_back(getAL(CurDAG)); // Predicate 3366 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register 3367 return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops.data(), Ops.size()); 3368 } 3369 case ARMISD::VTBL2: { 3370 DebugLoc dl = N->getDebugLoc(); 3371 EVT VT = N->getValueType(0); 3372 3373 // Form a REG_SEQUENCE to force register allocation. 3374 SDValue V0 = N->getOperand(0); 3375 SDValue V1 = N->getOperand(1); 3376 SDValue RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0); 3377 3378 SmallVector<SDValue, 6> Ops; 3379 Ops.push_back(RegSeq); 3380 Ops.push_back(N->getOperand(2)); 3381 Ops.push_back(getAL(CurDAG)); // Predicate 3382 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register 3383 return CurDAG->getMachineNode(ARM::VTBL2, dl, VT, 3384 Ops.data(), Ops.size()); 3385 } 3386 3387 case ISD::CONCAT_VECTORS: 3388 return SelectConcatVector(N); 3389 3390 case ARMISD::ATOMOR64_DAG: 3391 return SelectAtomic64(N, ARM::ATOMOR6432); 3392 case ARMISD::ATOMXOR64_DAG: 3393 return SelectAtomic64(N, ARM::ATOMXOR6432); 3394 case ARMISD::ATOMADD64_DAG: 3395 return SelectAtomic64(N, ARM::ATOMADD6432); 3396 case ARMISD::ATOMSUB64_DAG: 3397 return SelectAtomic64(N, ARM::ATOMSUB6432); 3398 case ARMISD::ATOMNAND64_DAG: 3399 return SelectAtomic64(N, ARM::ATOMNAND6432); 3400 case ARMISD::ATOMAND64_DAG: 3401 return SelectAtomic64(N, ARM::ATOMAND6432); 3402 case ARMISD::ATOMSWAP64_DAG: 3403 return SelectAtomic64(N, ARM::ATOMSWAP6432); 3404 case ARMISD::ATOMCMPXCHG64_DAG: 3405 return SelectAtomic64(N, ARM::ATOMCMPXCHG6432); 3406 } 3407 3408 return SelectCode(N); 3409 } 3410 3411 bool ARMDAGToDAGISel:: 3412 SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, 3413 std::vector<SDValue> &OutOps) { 3414 assert(ConstraintCode == 'm' && "unexpected asm memory constraint"); 3415 // Require the address to be in a register. That is safe for all ARM 3416 // variants and it is hard to do anything much smarter without knowing 3417 // how the operand is used. 3418 OutOps.push_back(Op); 3419 return false; 3420 } 3421 3422 /// createARMISelDag - This pass converts a legalized DAG into a 3423 /// ARM-specific DAG, ready for instruction scheduling. 3424 /// 3425 FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM, 3426 CodeGenOpt::Level OptLevel) { 3427 return new ARMDAGToDAGISel(TM, OptLevel); 3428 } 3429