1 //===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation ------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file defines the interfaces that BPF uses to lower LLVM code into a 11 // selection DAG. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "BPFISelLowering.h" 16 #include "BPF.h" 17 #include "BPFSubtarget.h" 18 #include "BPFTargetMachine.h" 19 #include "llvm/CodeGen/CallingConvLower.h" 20 #include "llvm/CodeGen/MachineFrameInfo.h" 21 #include "llvm/CodeGen/MachineFunction.h" 22 #include "llvm/CodeGen/MachineInstrBuilder.h" 23 #include "llvm/CodeGen/MachineRegisterInfo.h" 24 #include "llvm/CodeGen/SelectionDAGISel.h" 25 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 26 #include "llvm/CodeGen/ValueTypes.h" 27 #include "llvm/IR/DiagnosticInfo.h" 28 #include "llvm/IR/DiagnosticPrinter.h" 29 #include "llvm/Support/Debug.h" 30 #include "llvm/Support/ErrorHandling.h" 31 #include "llvm/Support/raw_ostream.h" 32 using namespace llvm; 33 34 #define DEBUG_TYPE "bpf-lower" 35 36 static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg) { 37 MachineFunction &MF = DAG.getMachineFunction(); 38 DAG.getContext()->diagnose( 39 DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc())); 40 } 41 42 static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg, 43 SDValue Val) { 44 MachineFunction &MF = DAG.getMachineFunction(); 45 std::string Str; 46 raw_string_ostream OS(Str); 47 OS << Msg; 48 Val->print(OS); 49 OS.flush(); 50 DAG.getContext()->diagnose( 51 DiagnosticInfoUnsupported(MF.getFunction(), Str, DL.getDebugLoc())); 52 } 53 54 BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, 55 const BPFSubtarget &STI) 56 : TargetLowering(TM) { 57 58 // Set up the register classes. 59 addRegisterClass(MVT::i64, &BPF::GPRRegClass); 60 if (STI.getHasAlu32()) 61 addRegisterClass(MVT::i32, &BPF::GPR32RegClass); 62 63 // Compute derived properties from the register classes 64 computeRegisterProperties(STI.getRegisterInfo()); 65 66 setStackPointerRegisterToSaveRestore(BPF::R11); 67 68 setOperationAction(ISD::BR_CC, MVT::i64, Custom); 69 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 70 setOperationAction(ISD::BRIND, MVT::Other, Expand); 71 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 72 73 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); 74 75 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom); 76 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 77 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 78 79 for (auto VT : { MVT::i32, MVT::i64 }) { 80 if (VT == MVT::i32 && !STI.getHasAlu32()) 81 continue; 82 83 setOperationAction(ISD::SDIVREM, VT, Expand); 84 setOperationAction(ISD::UDIVREM, VT, Expand); 85 setOperationAction(ISD::SREM, VT, Expand); 86 setOperationAction(ISD::UREM, VT, Expand); 87 setOperationAction(ISD::MULHU, VT, Expand); 88 setOperationAction(ISD::MULHS, VT, Expand); 89 setOperationAction(ISD::UMUL_LOHI, VT, Expand); 90 setOperationAction(ISD::SMUL_LOHI, VT, Expand); 91 setOperationAction(ISD::ADDC, VT, Expand); 92 setOperationAction(ISD::ADDE, VT, Expand); 93 setOperationAction(ISD::SUBC, VT, Expand); 94 setOperationAction(ISD::SUBE, VT, Expand); 95 setOperationAction(ISD::ROTR, VT, Expand); 96 setOperationAction(ISD::ROTL, VT, Expand); 97 setOperationAction(ISD::SHL_PARTS, VT, Expand); 98 setOperationAction(ISD::SRL_PARTS, VT, Expand); 99 setOperationAction(ISD::SRA_PARTS, VT, Expand); 100 setOperationAction(ISD::CTPOP, VT, Expand); 101 102 setOperationAction(ISD::SETCC, VT, Expand); 103 setOperationAction(ISD::SELECT, VT, Expand); 104 setOperationAction(ISD::SELECT_CC, VT, Custom); 105 } 106 107 if (STI.getHasAlu32()) { 108 setOperationAction(ISD::BSWAP, MVT::i32, Promote); 109 setOperationAction(ISD::BR_CC, MVT::i32, Promote); 110 } 111 112 setOperationAction(ISD::CTTZ, MVT::i64, Custom); 113 setOperationAction(ISD::CTLZ, MVT::i64, Custom); 114 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Custom); 115 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom); 116 117 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 118 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand); 119 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand); 120 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Expand); 121 122 // Extended load operations for i1 types must be promoted 123 for (MVT VT : MVT::integer_valuetypes()) { 124 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote); 125 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 126 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 127 128 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Expand); 129 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Expand); 130 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand); 131 } 132 133 setBooleanContents(ZeroOrOneBooleanContent); 134 135 // Function alignments (log2) 136 setMinFunctionAlignment(3); 137 setPrefFunctionAlignment(3); 138 139 // inline memcpy() for kernel to see explicit copy 140 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = 128; 141 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = 128; 142 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = 128; 143 144 // CPU/Feature control 145 HasAlu32 = STI.getHasAlu32(); 146 HasJmpExt = STI.getHasJmpExt(); 147 } 148 149 bool BPFTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { 150 return false; 151 } 152 153 std::pair<unsigned, const TargetRegisterClass *> 154 BPFTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, 155 StringRef Constraint, 156 MVT VT) const { 157 if (Constraint.size() == 1) 158 // GCC Constraint Letters 159 switch (Constraint[0]) { 160 case 'r': // GENERAL_REGS 161 return std::make_pair(0U, &BPF::GPRRegClass); 162 default: 163 break; 164 } 165 166 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 167 } 168 169 SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 170 switch (Op.getOpcode()) { 171 case ISD::BR_CC: 172 return LowerBR_CC(Op, DAG); 173 case ISD::GlobalAddress: 174 return LowerGlobalAddress(Op, DAG); 175 case ISD::SELECT_CC: 176 return LowerSELECT_CC(Op, DAG); 177 default: 178 llvm_unreachable("unimplemented operand"); 179 } 180 } 181 182 // Calling Convention Implementation 183 #include "BPFGenCallingConv.inc" 184 185 SDValue BPFTargetLowering::LowerFormalArguments( 186 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 187 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 188 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 189 switch (CallConv) { 190 default: 191 report_fatal_error("Unsupported calling convention"); 192 case CallingConv::C: 193 case CallingConv::Fast: 194 break; 195 } 196 197 MachineFunction &MF = DAG.getMachineFunction(); 198 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 199 200 // Assign locations to all of the incoming arguments. 201 SmallVector<CCValAssign, 16> ArgLocs; 202 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 203 CCInfo.AnalyzeFormalArguments(Ins, getHasAlu32() ? CC_BPF32 : CC_BPF64); 204 205 for (auto &VA : ArgLocs) { 206 if (VA.isRegLoc()) { 207 // Arguments passed in registers 208 EVT RegVT = VA.getLocVT(); 209 MVT::SimpleValueType SimpleTy = RegVT.getSimpleVT().SimpleTy; 210 switch (SimpleTy) { 211 default: { 212 errs() << "LowerFormalArguments Unhandled argument type: " 213 << RegVT.getEVTString() << '\n'; 214 llvm_unreachable(0); 215 } 216 case MVT::i32: 217 case MVT::i64: 218 unsigned VReg = RegInfo.createVirtualRegister(SimpleTy == MVT::i64 ? 219 &BPF::GPRRegClass : 220 &BPF::GPR32RegClass); 221 RegInfo.addLiveIn(VA.getLocReg(), VReg); 222 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, RegVT); 223 224 // If this is an value that has been promoted to wider types, insert an 225 // assert[sz]ext to capture this, then truncate to the right size. 226 if (VA.getLocInfo() == CCValAssign::SExt) 227 ArgValue = DAG.getNode(ISD::AssertSext, DL, RegVT, ArgValue, 228 DAG.getValueType(VA.getValVT())); 229 else if (VA.getLocInfo() == CCValAssign::ZExt) 230 ArgValue = DAG.getNode(ISD::AssertZext, DL, RegVT, ArgValue, 231 DAG.getValueType(VA.getValVT())); 232 233 if (VA.getLocInfo() != CCValAssign::Full) 234 ArgValue = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), ArgValue); 235 236 InVals.push_back(ArgValue); 237 238 break; 239 } 240 } else { 241 fail(DL, DAG, "defined with too many args"); 242 InVals.push_back(DAG.getConstant(0, DL, VA.getLocVT())); 243 } 244 } 245 246 if (IsVarArg || MF.getFunction().hasStructRetAttr()) { 247 fail(DL, DAG, "functions with VarArgs or StructRet are not supported"); 248 } 249 250 return Chain; 251 } 252 253 const unsigned BPFTargetLowering::MaxArgs = 5; 254 255 SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, 256 SmallVectorImpl<SDValue> &InVals) const { 257 SelectionDAG &DAG = CLI.DAG; 258 auto &Outs = CLI.Outs; 259 auto &OutVals = CLI.OutVals; 260 auto &Ins = CLI.Ins; 261 SDValue Chain = CLI.Chain; 262 SDValue Callee = CLI.Callee; 263 bool &IsTailCall = CLI.IsTailCall; 264 CallingConv::ID CallConv = CLI.CallConv; 265 bool IsVarArg = CLI.IsVarArg; 266 MachineFunction &MF = DAG.getMachineFunction(); 267 268 // BPF target does not support tail call optimization. 269 IsTailCall = false; 270 271 switch (CallConv) { 272 default: 273 report_fatal_error("Unsupported calling convention"); 274 case CallingConv::Fast: 275 case CallingConv::C: 276 break; 277 } 278 279 // Analyze operands of the call, assigning locations to each operand. 280 SmallVector<CCValAssign, 16> ArgLocs; 281 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 282 283 CCInfo.AnalyzeCallOperands(Outs, getHasAlu32() ? CC_BPF32 : CC_BPF64); 284 285 unsigned NumBytes = CCInfo.getNextStackOffset(); 286 287 if (Outs.size() > MaxArgs) 288 fail(CLI.DL, DAG, "too many args to ", Callee); 289 290 for (auto &Arg : Outs) { 291 ISD::ArgFlagsTy Flags = Arg.Flags; 292 if (!Flags.isByVal()) 293 continue; 294 295 fail(CLI.DL, DAG, "pass by value not supported ", Callee); 296 } 297 298 auto PtrVT = getPointerTy(MF.getDataLayout()); 299 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL); 300 301 SmallVector<std::pair<unsigned, SDValue>, MaxArgs> RegsToPass; 302 303 // Walk arg assignments 304 for (unsigned i = 0, 305 e = std::min(static_cast<unsigned>(ArgLocs.size()), MaxArgs); 306 i != e; ++i) { 307 CCValAssign &VA = ArgLocs[i]; 308 SDValue Arg = OutVals[i]; 309 310 // Promote the value if needed. 311 switch (VA.getLocInfo()) { 312 default: 313 llvm_unreachable("Unknown loc info"); 314 case CCValAssign::Full: 315 break; 316 case CCValAssign::SExt: 317 Arg = DAG.getNode(ISD::SIGN_EXTEND, CLI.DL, VA.getLocVT(), Arg); 318 break; 319 case CCValAssign::ZExt: 320 Arg = DAG.getNode(ISD::ZERO_EXTEND, CLI.DL, VA.getLocVT(), Arg); 321 break; 322 case CCValAssign::AExt: 323 Arg = DAG.getNode(ISD::ANY_EXTEND, CLI.DL, VA.getLocVT(), Arg); 324 break; 325 } 326 327 // Push arguments into RegsToPass vector 328 if (VA.isRegLoc()) 329 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 330 else 331 llvm_unreachable("call arg pass bug"); 332 } 333 334 SDValue InFlag; 335 336 // Build a sequence of copy-to-reg nodes chained together with token chain and 337 // flag operands which copy the outgoing args into registers. The InFlag in 338 // necessary since all emitted instructions must be stuck together. 339 for (auto &Reg : RegsToPass) { 340 Chain = DAG.getCopyToReg(Chain, CLI.DL, Reg.first, Reg.second, InFlag); 341 InFlag = Chain.getValue(1); 342 } 343 344 // If the callee is a GlobalAddress node (quite common, every direct call is) 345 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. 346 // Likewise ExternalSymbol -> TargetExternalSymbol. 347 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 348 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), CLI.DL, PtrVT, 349 G->getOffset(), 0); 350 } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { 351 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0); 352 fail(CLI.DL, DAG, Twine("A call to built-in function '" 353 + StringRef(E->getSymbol()) 354 + "' is not supported.")); 355 } 356 357 // Returns a chain & a flag for retval copy to use. 358 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 359 SmallVector<SDValue, 8> Ops; 360 Ops.push_back(Chain); 361 Ops.push_back(Callee); 362 363 // Add argument registers to the end of the list so that they are 364 // known live into the call. 365 for (auto &Reg : RegsToPass) 366 Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType())); 367 368 if (InFlag.getNode()) 369 Ops.push_back(InFlag); 370 371 Chain = DAG.getNode(BPFISD::CALL, CLI.DL, NodeTys, Ops); 372 InFlag = Chain.getValue(1); 373 374 // Create the CALLSEQ_END node. 375 Chain = DAG.getCALLSEQ_END( 376 Chain, DAG.getConstant(NumBytes, CLI.DL, PtrVT, true), 377 DAG.getConstant(0, CLI.DL, PtrVT, true), InFlag, CLI.DL); 378 InFlag = Chain.getValue(1); 379 380 // Handle result values, copying them out of physregs into vregs that we 381 // return. 382 return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, CLI.DL, DAG, 383 InVals); 384 } 385 386 SDValue 387 BPFTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 388 bool IsVarArg, 389 const SmallVectorImpl<ISD::OutputArg> &Outs, 390 const SmallVectorImpl<SDValue> &OutVals, 391 const SDLoc &DL, SelectionDAG &DAG) const { 392 unsigned Opc = BPFISD::RET_FLAG; 393 394 // CCValAssign - represent the assignment of the return value to a location 395 SmallVector<CCValAssign, 16> RVLocs; 396 MachineFunction &MF = DAG.getMachineFunction(); 397 398 // CCState - Info about the registers and stack slot. 399 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext()); 400 401 if (MF.getFunction().getReturnType()->isAggregateType()) { 402 fail(DL, DAG, "only integer returns supported"); 403 return DAG.getNode(Opc, DL, MVT::Other, Chain); 404 } 405 406 // Analize return values. 407 CCInfo.AnalyzeReturn(Outs, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64); 408 409 SDValue Flag; 410 SmallVector<SDValue, 4> RetOps(1, Chain); 411 412 // Copy the result values into the output registers. 413 for (unsigned i = 0; i != RVLocs.size(); ++i) { 414 CCValAssign &VA = RVLocs[i]; 415 assert(VA.isRegLoc() && "Can only return in registers!"); 416 417 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), OutVals[i], Flag); 418 419 // Guarantee that all emitted copies are stuck together, 420 // avoiding something bad. 421 Flag = Chain.getValue(1); 422 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 423 } 424 425 RetOps[0] = Chain; // Update chain. 426 427 // Add the flag if we have it. 428 if (Flag.getNode()) 429 RetOps.push_back(Flag); 430 431 return DAG.getNode(Opc, DL, MVT::Other, RetOps); 432 } 433 434 SDValue BPFTargetLowering::LowerCallResult( 435 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg, 436 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 437 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 438 439 MachineFunction &MF = DAG.getMachineFunction(); 440 // Assign locations to each value returned by this call. 441 SmallVector<CCValAssign, 16> RVLocs; 442 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext()); 443 444 if (Ins.size() >= 2) { 445 fail(DL, DAG, "only small returns supported"); 446 for (unsigned i = 0, e = Ins.size(); i != e; ++i) 447 InVals.push_back(DAG.getConstant(0, DL, Ins[i].VT)); 448 return DAG.getCopyFromReg(Chain, DL, 1, Ins[0].VT, InFlag).getValue(1); 449 } 450 451 CCInfo.AnalyzeCallResult(Ins, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64); 452 453 // Copy all of the result registers out of their specified physreg. 454 for (auto &Val : RVLocs) { 455 Chain = DAG.getCopyFromReg(Chain, DL, Val.getLocReg(), 456 Val.getValVT(), InFlag).getValue(1); 457 InFlag = Chain.getValue(2); 458 InVals.push_back(Chain.getValue(0)); 459 } 460 461 return Chain; 462 } 463 464 static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) { 465 switch (CC) { 466 default: 467 break; 468 case ISD::SETULT: 469 case ISD::SETULE: 470 case ISD::SETLT: 471 case ISD::SETLE: 472 CC = ISD::getSetCCSwappedOperands(CC); 473 std::swap(LHS, RHS); 474 break; 475 } 476 } 477 478 SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 479 SDValue Chain = Op.getOperand(0); 480 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 481 SDValue LHS = Op.getOperand(2); 482 SDValue RHS = Op.getOperand(3); 483 SDValue Dest = Op.getOperand(4); 484 SDLoc DL(Op); 485 486 if (!getHasJmpExt()) 487 NegateCC(LHS, RHS, CC); 488 489 return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS, 490 DAG.getConstant(CC, DL, MVT::i64), Dest); 491 } 492 493 SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 494 SDValue LHS = Op.getOperand(0); 495 SDValue RHS = Op.getOperand(1); 496 SDValue TrueV = Op.getOperand(2); 497 SDValue FalseV = Op.getOperand(3); 498 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 499 SDLoc DL(Op); 500 501 if (!getHasJmpExt()) 502 NegateCC(LHS, RHS, CC); 503 504 SDValue TargetCC = DAG.getConstant(CC, DL, LHS.getValueType()); 505 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 506 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV}; 507 508 return DAG.getNode(BPFISD::SELECT_CC, DL, VTs, Ops); 509 } 510 511 const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const { 512 switch ((BPFISD::NodeType)Opcode) { 513 case BPFISD::FIRST_NUMBER: 514 break; 515 case BPFISD::RET_FLAG: 516 return "BPFISD::RET_FLAG"; 517 case BPFISD::CALL: 518 return "BPFISD::CALL"; 519 case BPFISD::SELECT_CC: 520 return "BPFISD::SELECT_CC"; 521 case BPFISD::BR_CC: 522 return "BPFISD::BR_CC"; 523 case BPFISD::Wrapper: 524 return "BPFISD::Wrapper"; 525 } 526 return nullptr; 527 } 528 529 SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op, 530 SelectionDAG &DAG) const { 531 auto N = cast<GlobalAddressSDNode>(Op); 532 assert(N->getOffset() == 0 && "Invalid offset for global address"); 533 534 SDLoc DL(Op); 535 const GlobalValue *GV = N->getGlobal(); 536 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i64); 537 538 return DAG.getNode(BPFISD::Wrapper, DL, MVT::i64, GA); 539 } 540 541 unsigned 542 BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB, 543 unsigned Reg, bool isSigned) const { 544 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo(); 545 const TargetRegisterClass *RC = getRegClassFor(MVT::i64); 546 int RShiftOp = isSigned ? BPF::SRA_ri : BPF::SRL_ri; 547 MachineFunction *F = BB->getParent(); 548 DebugLoc DL = MI.getDebugLoc(); 549 550 MachineRegisterInfo &RegInfo = F->getRegInfo(); 551 unsigned PromotedReg0 = RegInfo.createVirtualRegister(RC); 552 unsigned PromotedReg1 = RegInfo.createVirtualRegister(RC); 553 unsigned PromotedReg2 = RegInfo.createVirtualRegister(RC); 554 BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg); 555 BuildMI(BB, DL, TII.get(BPF::SLL_ri), PromotedReg1) 556 .addReg(PromotedReg0).addImm(32); 557 BuildMI(BB, DL, TII.get(RShiftOp), PromotedReg2) 558 .addReg(PromotedReg1).addImm(32); 559 560 return PromotedReg2; 561 } 562 563 MachineBasicBlock * 564 BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, 565 MachineBasicBlock *BB) const { 566 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo(); 567 DebugLoc DL = MI.getDebugLoc(); 568 unsigned Opc = MI.getOpcode(); 569 bool isSelectRROp = (Opc == BPF::Select || 570 Opc == BPF::Select_64_32 || 571 Opc == BPF::Select_32 || 572 Opc == BPF::Select_32_64); 573 574 #ifndef NDEBUG 575 bool isSelectRIOp = (Opc == BPF::Select_Ri || 576 Opc == BPF::Select_Ri_64_32 || 577 Opc == BPF::Select_Ri_32 || 578 Opc == BPF::Select_Ri_32_64); 579 580 581 assert((isSelectRROp || isSelectRIOp) && "Unexpected instr type to insert"); 582 #endif 583 584 bool is32BitCmp = (Opc == BPF::Select_32 || 585 Opc == BPF::Select_32_64 || 586 Opc == BPF::Select_Ri_32 || 587 Opc == BPF::Select_Ri_32_64); 588 589 // To "insert" a SELECT instruction, we actually have to insert the diamond 590 // control-flow pattern. The incoming instruction knows the destination vreg 591 // to set, the condition code register to branch on, the true/false values to 592 // select between, and a branch opcode to use. 593 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 594 MachineFunction::iterator I = ++BB->getIterator(); 595 596 // ThisMBB: 597 // ... 598 // TrueVal = ... 599 // jmp_XX r1, r2 goto Copy1MBB 600 // fallthrough --> Copy0MBB 601 MachineBasicBlock *ThisMBB = BB; 602 MachineFunction *F = BB->getParent(); 603 MachineBasicBlock *Copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 604 MachineBasicBlock *Copy1MBB = F->CreateMachineBasicBlock(LLVM_BB); 605 606 F->insert(I, Copy0MBB); 607 F->insert(I, Copy1MBB); 608 // Update machine-CFG edges by transferring all successors of the current 609 // block to the new block which will contain the Phi node for the select. 610 Copy1MBB->splice(Copy1MBB->begin(), BB, 611 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 612 Copy1MBB->transferSuccessorsAndUpdatePHIs(BB); 613 // Next, add the true and fallthrough blocks as its successors. 614 BB->addSuccessor(Copy0MBB); 615 BB->addSuccessor(Copy1MBB); 616 617 // Insert Branch if Flag 618 int CC = MI.getOperand(3).getImm(); 619 int NewCC; 620 switch (CC) { 621 case ISD::SETGT: 622 NewCC = isSelectRROp ? BPF::JSGT_rr : BPF::JSGT_ri; 623 break; 624 case ISD::SETUGT: 625 NewCC = isSelectRROp ? BPF::JUGT_rr : BPF::JUGT_ri; 626 break; 627 case ISD::SETGE: 628 NewCC = isSelectRROp ? BPF::JSGE_rr : BPF::JSGE_ri; 629 break; 630 case ISD::SETUGE: 631 NewCC = isSelectRROp ? BPF::JUGE_rr : BPF::JUGE_ri; 632 break; 633 case ISD::SETEQ: 634 NewCC = isSelectRROp ? BPF::JEQ_rr : BPF::JEQ_ri; 635 break; 636 case ISD::SETNE: 637 NewCC = isSelectRROp ? BPF::JNE_rr : BPF::JNE_ri; 638 break; 639 case ISD::SETLT: 640 NewCC = isSelectRROp ? BPF::JSLT_rr : BPF::JSLT_ri; 641 break; 642 case ISD::SETULT: 643 NewCC = isSelectRROp ? BPF::JULT_rr : BPF::JULT_ri; 644 break; 645 case ISD::SETLE: 646 NewCC = isSelectRROp ? BPF::JSLE_rr : BPF::JSLE_ri; 647 break; 648 case ISD::SETULE: 649 NewCC = isSelectRROp ? BPF::JULE_rr : BPF::JULE_ri; 650 break; 651 default: 652 report_fatal_error("unimplemented select CondCode " + Twine(CC)); 653 } 654 655 unsigned LHS = MI.getOperand(1).getReg(); 656 bool isSignedCmp = (CC == ISD::SETGT || 657 CC == ISD::SETGE || 658 CC == ISD::SETLT || 659 CC == ISD::SETLE); 660 661 // eBPF at the moment only has 64-bit comparison. Any 32-bit comparison need 662 // to be promoted, however if the 32-bit comparison operands are destination 663 // registers then they are implicitly zero-extended already, there is no 664 // need of explicit zero-extend sequence for them. 665 // 666 // We simply do extension for all situations in this method, but we will 667 // try to remove those unnecessary in BPFMIPeephole pass. 668 if (is32BitCmp) 669 LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp); 670 671 if (isSelectRROp) { 672 unsigned RHS = MI.getOperand(2).getReg(); 673 674 if (is32BitCmp) 675 RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp); 676 677 BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB); 678 } else { 679 int64_t imm32 = MI.getOperand(2).getImm(); 680 // sanity check before we build J*_ri instruction. 681 assert (isInt<32>(imm32)); 682 BuildMI(BB, DL, TII.get(NewCC)) 683 .addReg(LHS).addImm(imm32).addMBB(Copy1MBB); 684 } 685 686 // Copy0MBB: 687 // %FalseValue = ... 688 // # fallthrough to Copy1MBB 689 BB = Copy0MBB; 690 691 // Update machine-CFG edges 692 BB->addSuccessor(Copy1MBB); 693 694 // Copy1MBB: 695 // %Result = phi [ %FalseValue, Copy0MBB ], [ %TrueValue, ThisMBB ] 696 // ... 697 BB = Copy1MBB; 698 BuildMI(*BB, BB->begin(), DL, TII.get(BPF::PHI), MI.getOperand(0).getReg()) 699 .addReg(MI.getOperand(5).getReg()) 700 .addMBB(Copy0MBB) 701 .addReg(MI.getOperand(4).getReg()) 702 .addMBB(ThisMBB); 703 704 MI.eraseFromParent(); // The pseudo instruction is gone now. 705 return BB; 706 } 707 708 EVT BPFTargetLowering::getSetCCResultType(const DataLayout &, LLVMContext &, 709 EVT VT) const { 710 return getHasAlu32() ? MVT::i32 : MVT::i64; 711 } 712 713 MVT BPFTargetLowering::getScalarShiftAmountTy(const DataLayout &DL, 714 EVT VT) const { 715 return (getHasAlu32() && VT == MVT::i32) ? MVT::i32 : MVT::i64; 716 } 717