1 //===-- MipsExpandPseudoInsts.cpp - Expand pseudo instructions ------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains a pass that expands pseudo instructions into target
11 // instructions to allow proper scheduling, if-conversion, and other late
12 // optimizations. This pass should be run after register allocation but before
13 // the post-regalloc scheduling pass.
14 //
15 // This is currently only used for expanding atomic pseudos after register
16 // allocation. We do this to avoid the fast register allocator introducing
17 // spills between ll and sc. These stores cause some MIPS implementations to
18 // abort the atomic RMW sequence.
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #include "Mips.h"
23 #include "MipsInstrInfo.h"
24 #include "MipsSubtarget.h"
25 #include "llvm/CodeGen/LivePhysRegs.h"
26 #include "llvm/CodeGen/MachineFunctionPass.h"
27 #include "llvm/CodeGen/MachineInstrBuilder.h"
28 
29 using namespace llvm;
30 
31 #define DEBUG_TYPE "mips-pseudo"
32 
33 namespace {
34   class MipsExpandPseudo : public MachineFunctionPass {
35   public:
36     static char ID;
37     MipsExpandPseudo() : MachineFunctionPass(ID) {}
38 
39     const MipsInstrInfo *TII;
40     const MipsSubtarget *STI;
41 
42     bool runOnMachineFunction(MachineFunction &Fn) override;
43 
44     MachineFunctionProperties getRequiredProperties() const override {
45       return MachineFunctionProperties().set(
46           MachineFunctionProperties::Property::NoVRegs);
47     }
48 
49     StringRef getPassName() const override {
50       return "Mips pseudo instruction expansion pass";
51     }
52 
53   private:
54     bool expandAtomicCmpSwap(MachineBasicBlock &MBB,
55                              MachineBasicBlock::iterator MBBI,
56                              MachineBasicBlock::iterator &NextMBBI);
57     bool expandAtomicCmpSwapSubword(MachineBasicBlock &MBB,
58                                     MachineBasicBlock::iterator MBBI,
59                                     MachineBasicBlock::iterator &NextMBBI);
60 
61     bool expandAtomicBinOp(MachineBasicBlock &BB,
62                            MachineBasicBlock::iterator I,
63                            MachineBasicBlock::iterator &NMBBI, unsigned Size);
64     bool expandAtomicBinOpSubword(MachineBasicBlock &BB,
65                                   MachineBasicBlock::iterator I,
66                                   MachineBasicBlock::iterator &NMBBI);
67 
68     bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
69                   MachineBasicBlock::iterator &NMBB);
70     bool expandMBB(MachineBasicBlock &MBB);
71    };
72   char MipsExpandPseudo::ID = 0;
73 }
74 
75 bool MipsExpandPseudo::expandAtomicCmpSwapSubword(
76     MachineBasicBlock &BB, MachineBasicBlock::iterator I,
77     MachineBasicBlock::iterator &NMBBI) {
78 
79   MachineFunction *MF = BB.getParent();
80 
81   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
82   DebugLoc DL = I->getDebugLoc();
83   unsigned LL, SC;
84 
85   unsigned ZERO = Mips::ZERO;
86   unsigned BNE = Mips::BNE;
87   unsigned BEQ = Mips::BEQ;
88   unsigned SEOp =
89       I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB : Mips::SEH;
90 
91   if (STI->inMicroMipsMode()) {
92       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
93       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
94       BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
95       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
96   } else {
97     LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
98                             : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
99     SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
100                             : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
101   }
102 
103   unsigned Dest = I->getOperand(0).getReg();
104   unsigned Ptr = I->getOperand(1).getReg();
105   unsigned Mask = I->getOperand(2).getReg();
106   unsigned ShiftCmpVal = I->getOperand(3).getReg();
107   unsigned Mask2 = I->getOperand(4).getReg();
108   unsigned ShiftNewVal = I->getOperand(5).getReg();
109   unsigned ShiftAmnt = I->getOperand(6).getReg();
110   unsigned Scratch = I->getOperand(7).getReg();
111   unsigned Scratch2 = I->getOperand(8).getReg();
112 
113   // insert new blocks after the current block
114   const BasicBlock *LLVM_BB = BB.getBasicBlock();
115   MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
116   MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
117   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
118   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
119   MachineFunction::iterator It = ++BB.getIterator();
120   MF->insert(It, loop1MBB);
121   MF->insert(It, loop2MBB);
122   MF->insert(It, sinkMBB);
123   MF->insert(It, exitMBB);
124 
125   // Transfer the remainder of BB and its successor edges to exitMBB.
126   exitMBB->splice(exitMBB->begin(), &BB,
127                   std::next(MachineBasicBlock::iterator(I)), BB.end());
128   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
129 
130   //  thisMBB:
131   //    ...
132   //    fallthrough --> loop1MBB
133   BB.addSuccessor(loop1MBB, BranchProbability::getOne());
134   loop1MBB->addSuccessor(sinkMBB);
135   loop1MBB->addSuccessor(loop2MBB);
136   loop1MBB->normalizeSuccProbs();
137   loop2MBB->addSuccessor(loop1MBB);
138   loop2MBB->addSuccessor(sinkMBB);
139   loop2MBB->normalizeSuccProbs();
140   sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
141 
142   // loop1MBB:
143   //   ll dest, 0(ptr)
144   //   and Mask', dest, Mask
145   //   bne Mask', ShiftCmpVal, exitMBB
146   BuildMI(loop1MBB, DL, TII->get(LL), Scratch).addReg(Ptr).addImm(0);
147   BuildMI(loop1MBB, DL, TII->get(Mips::AND), Scratch2)
148       .addReg(Scratch)
149       .addReg(Mask);
150   BuildMI(loop1MBB, DL, TII->get(BNE))
151     .addReg(Scratch2).addReg(ShiftCmpVal).addMBB(sinkMBB);
152 
153   // loop2MBB:
154   //   and dest, dest, mask2
155   //   or dest, dest, ShiftNewVal
156   //   sc dest, dest, 0(ptr)
157   //   beq dest, $0, loop1MBB
158   BuildMI(loop2MBB, DL, TII->get(Mips::AND), Scratch)
159       .addReg(Scratch, RegState::Kill)
160       .addReg(Mask2);
161   BuildMI(loop2MBB, DL, TII->get(Mips::OR), Scratch)
162       .addReg(Scratch, RegState::Kill)
163       .addReg(ShiftNewVal);
164   BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
165       .addReg(Scratch, RegState::Kill)
166       .addReg(Ptr)
167       .addImm(0);
168   BuildMI(loop2MBB, DL, TII->get(BEQ))
169       .addReg(Scratch, RegState::Kill)
170       .addReg(ZERO)
171       .addMBB(loop1MBB);
172 
173   //  sinkMBB:
174   //    srl     srlres, Mask', shiftamt
175   //    sign_extend dest,srlres
176   BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
177       .addReg(Scratch2)
178       .addReg(ShiftAmnt);
179   if (STI->hasMips32r2()) {
180     BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
181   } else {
182     const unsigned ShiftImm =
183         I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 16 : 24;
184     BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
185         .addReg(Dest, RegState::Kill)
186         .addImm(ShiftImm);
187     BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
188         .addReg(Dest, RegState::Kill)
189         .addImm(ShiftImm);
190   }
191 
192   LivePhysRegs LiveRegs;
193   computeAndAddLiveIns(LiveRegs, *loop1MBB);
194   computeAndAddLiveIns(LiveRegs, *loop2MBB);
195   computeAndAddLiveIns(LiveRegs, *sinkMBB);
196   computeAndAddLiveIns(LiveRegs, *exitMBB);
197 
198   NMBBI = BB.end();
199   I->eraseFromParent();
200   return true;
201 }
202 
203 bool MipsExpandPseudo::expandAtomicCmpSwap(MachineBasicBlock &BB,
204                                            MachineBasicBlock::iterator I,
205                                            MachineBasicBlock::iterator &NMBBI) {
206 
207   const unsigned Size =
208       I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 4 : 8;
209   MachineFunction *MF = BB.getParent();
210 
211   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
212   DebugLoc DL = I->getDebugLoc();
213 
214   unsigned LL, SC, ZERO, BNE, BEQ, MOVE;
215 
216   if (Size == 4) {
217     if (STI->inMicroMipsMode()) {
218       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
219       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
220       BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
221       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
222     } else {
223       LL = STI->hasMips32r6()
224                ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
225                : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
226       SC = STI->hasMips32r6()
227                ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
228                : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
229       BNE = Mips::BNE;
230       BEQ = Mips::BEQ;
231     }
232 
233     ZERO = Mips::ZERO;
234     MOVE = Mips::OR;
235   } else {
236     LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
237     SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
238     ZERO = Mips::ZERO_64;
239     BNE = Mips::BNE64;
240     BEQ = Mips::BEQ64;
241     MOVE = Mips::OR64;
242   }
243 
244   unsigned Dest = I->getOperand(0).getReg();
245   unsigned Ptr = I->getOperand(1).getReg();
246   unsigned OldVal = I->getOperand(2).getReg();
247   unsigned NewVal = I->getOperand(3).getReg();
248   unsigned Scratch = I->getOperand(4).getReg();
249 
250   // insert new blocks after the current block
251   const BasicBlock *LLVM_BB = BB.getBasicBlock();
252   MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
253   MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
254   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
255   MachineFunction::iterator It = ++BB.getIterator();
256   MF->insert(It, loop1MBB);
257   MF->insert(It, loop2MBB);
258   MF->insert(It, exitMBB);
259 
260   // Transfer the remainder of BB and its successor edges to exitMBB.
261   exitMBB->splice(exitMBB->begin(), &BB,
262                   std::next(MachineBasicBlock::iterator(I)), BB.end());
263   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
264 
265   //  thisMBB:
266   //    ...
267   //    fallthrough --> loop1MBB
268   BB.addSuccessor(loop1MBB, BranchProbability::getOne());
269   loop1MBB->addSuccessor(exitMBB);
270   loop1MBB->addSuccessor(loop2MBB);
271   loop1MBB->normalizeSuccProbs();
272   loop2MBB->addSuccessor(loop1MBB);
273   loop2MBB->addSuccessor(exitMBB);
274   loop2MBB->normalizeSuccProbs();
275 
276   // loop1MBB:
277   //   ll dest, 0(ptr)
278   //   bne dest, oldval, exitMBB
279   BuildMI(loop1MBB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0);
280   BuildMI(loop1MBB, DL, TII->get(BNE))
281     .addReg(Dest, RegState::Kill).addReg(OldVal).addMBB(exitMBB);
282 
283   // loop2MBB:
284   //   move scratch, NewVal
285   //   sc Scratch, Scratch, 0(ptr)
286   //   beq Scratch, $0, loop1MBB
287   BuildMI(loop2MBB, DL, TII->get(MOVE), Scratch).addReg(NewVal).addReg(ZERO);
288   BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
289     .addReg(Scratch).addReg(Ptr).addImm(0);
290   BuildMI(loop2MBB, DL, TII->get(BEQ))
291     .addReg(Scratch, RegState::Kill).addReg(ZERO).addMBB(loop1MBB);
292 
293   LivePhysRegs LiveRegs;
294   computeAndAddLiveIns(LiveRegs, *loop1MBB);
295   computeAndAddLiveIns(LiveRegs, *loop2MBB);
296   computeAndAddLiveIns(LiveRegs, *exitMBB);
297 
298   NMBBI = BB.end();
299   I->eraseFromParent();
300   return true;
301 }
302 
303 bool MipsExpandPseudo::expandAtomicBinOpSubword(
304     MachineBasicBlock &BB, MachineBasicBlock::iterator I,
305     MachineBasicBlock::iterator &NMBBI) {
306 
307   MachineFunction *MF = BB.getParent();
308 
309   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
310   DebugLoc DL = I->getDebugLoc();
311 
312   unsigned LL, SC;
313   unsigned BEQ = Mips::BEQ;
314   unsigned SEOp = Mips::SEH;
315 
316   if (STI->inMicroMipsMode()) {
317       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
318       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
319       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
320   } else {
321     LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
322                             : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
323     SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
324                             : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
325   }
326 
327   bool IsSwap = false;
328   bool IsNand = false;
329 
330   unsigned Opcode = 0;
331   switch (I->getOpcode()) {
332   case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
333     SEOp = Mips::SEB;
334     LLVM_FALLTHROUGH;
335   case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
336     IsNand = true;
337     break;
338   case Mips::ATOMIC_SWAP_I8_POSTRA:
339     SEOp = Mips::SEB;
340     LLVM_FALLTHROUGH;
341   case Mips::ATOMIC_SWAP_I16_POSTRA:
342     IsSwap = true;
343     break;
344   case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
345     SEOp = Mips::SEB;
346     LLVM_FALLTHROUGH;
347   case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
348     Opcode = Mips::ADDu;
349     break;
350   case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
351     SEOp = Mips::SEB;
352     LLVM_FALLTHROUGH;
353   case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
354     Opcode = Mips::SUBu;
355     break;
356   case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
357     SEOp = Mips::SEB;
358     LLVM_FALLTHROUGH;
359   case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
360     Opcode = Mips::AND;
361     break;
362   case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
363     SEOp = Mips::SEB;
364     LLVM_FALLTHROUGH;
365   case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
366     Opcode = Mips::OR;
367     break;
368   case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
369     SEOp = Mips::SEB;
370     LLVM_FALLTHROUGH;
371   case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
372     Opcode = Mips::XOR;
373     break;
374   default:
375     llvm_unreachable("Unknown subword atomic pseudo for expansion!");
376   }
377 
378   unsigned Dest = I->getOperand(0).getReg();
379   unsigned Ptr = I->getOperand(1).getReg();
380   unsigned Incr = I->getOperand(2).getReg();
381   unsigned Mask = I->getOperand(3).getReg();
382   unsigned Mask2 = I->getOperand(4).getReg();
383   unsigned ShiftAmnt = I->getOperand(5).getReg();
384   unsigned OldVal = I->getOperand(6).getReg();
385   unsigned BinOpRes = I->getOperand(7).getReg();
386   unsigned StoreVal = I->getOperand(8).getReg();
387 
388   const BasicBlock *LLVM_BB = BB.getBasicBlock();
389   MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
390   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
391   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
392   MachineFunction::iterator It = ++BB.getIterator();
393   MF->insert(It, loopMBB);
394   MF->insert(It, sinkMBB);
395   MF->insert(It, exitMBB);
396 
397   exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
398   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
399 
400   BB.addSuccessor(loopMBB, BranchProbability::getOne());
401   loopMBB->addSuccessor(sinkMBB);
402   loopMBB->addSuccessor(loopMBB);
403   loopMBB->normalizeSuccProbs();
404 
405   BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
406   if (IsNand) {
407     //  and andres, oldval, incr2
408     //  nor binopres, $0, andres
409     //  and newval, binopres, mask
410     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
411         .addReg(OldVal)
412         .addReg(Incr);
413     BuildMI(loopMBB, DL, TII->get(Mips::NOR), BinOpRes)
414         .addReg(Mips::ZERO)
415         .addReg(BinOpRes);
416     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
417         .addReg(BinOpRes)
418         .addReg(Mask);
419   } else if (!IsSwap) {
420     //  <binop> binopres, oldval, incr2
421     //  and newval, binopres, mask
422     BuildMI(loopMBB, DL, TII->get(Opcode), BinOpRes)
423         .addReg(OldVal)
424         .addReg(Incr);
425     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
426         .addReg(BinOpRes)
427         .addReg(Mask);
428   } else { // atomic.swap
429     //  and newval, incr2, mask
430     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
431         .addReg(Incr)
432         .addReg(Mask);
433   }
434 
435   // and StoreVal, OlddVal, Mask2
436   // or StoreVal, StoreVal, BinOpRes
437   // StoreVal<tied1> = sc StoreVal, 0(Ptr)
438   // beq StoreVal, zero, loopMBB
439   BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal)
440     .addReg(OldVal).addReg(Mask2);
441   BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal)
442     .addReg(StoreVal).addReg(BinOpRes);
443   BuildMI(loopMBB, DL, TII->get(SC), StoreVal)
444     .addReg(StoreVal).addReg(Ptr).addImm(0);
445   BuildMI(loopMBB, DL, TII->get(BEQ))
446     .addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB);
447 
448   //  sinkMBB:
449   //    and     maskedoldval1,oldval,mask
450   //    srl     srlres,maskedoldval1,shiftamt
451   //    sign_extend dest,srlres
452 
453   sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
454 
455   BuildMI(sinkMBB, DL, TII->get(Mips::AND), Dest)
456     .addReg(OldVal).addReg(Mask);
457   BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
458       .addReg(Dest).addReg(ShiftAmnt);
459 
460   if (STI->hasMips32r2()) {
461     BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
462   } else {
463     const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
464     BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
465         .addReg(Dest, RegState::Kill)
466         .addImm(ShiftImm);
467     BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
468         .addReg(Dest, RegState::Kill)
469         .addImm(ShiftImm);
470   }
471 
472   LivePhysRegs LiveRegs;
473   computeAndAddLiveIns(LiveRegs, *loopMBB);
474   computeAndAddLiveIns(LiveRegs, *sinkMBB);
475   computeAndAddLiveIns(LiveRegs, *exitMBB);
476 
477   NMBBI = BB.end();
478   I->eraseFromParent();
479 
480   return true;
481 }
482 
483 bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB,
484                                          MachineBasicBlock::iterator I,
485                                          MachineBasicBlock::iterator &NMBBI,
486                                          unsigned Size) {
487   MachineFunction *MF = BB.getParent();
488 
489   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
490   DebugLoc DL = I->getDebugLoc();
491 
492   unsigned LL, SC, ZERO, BEQ;
493 
494   if (Size == 4) {
495     if (STI->inMicroMipsMode()) {
496       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
497       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
498       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
499     } else {
500       LL = STI->hasMips32r6()
501                ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
502                : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
503       SC = STI->hasMips32r6()
504                ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
505                : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
506       BEQ = Mips::BEQ;
507     }
508 
509     ZERO = Mips::ZERO;
510   } else {
511     LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
512     SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
513     ZERO = Mips::ZERO_64;
514     BEQ = Mips::BEQ64;
515   }
516 
517   unsigned OldVal = I->getOperand(0).getReg();
518   unsigned Ptr = I->getOperand(1).getReg();
519   unsigned Incr = I->getOperand(2).getReg();
520   unsigned Scratch = I->getOperand(3).getReg();
521 
522   unsigned Opcode = 0;
523   unsigned OR = 0;
524   unsigned AND = 0;
525   unsigned NOR = 0;
526   bool IsNand = false;
527   switch (I->getOpcode()) {
528   case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
529     Opcode = Mips::ADDu;
530     break;
531   case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
532     Opcode = Mips::SUBu;
533     break;
534   case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
535     Opcode = Mips::AND;
536     break;
537   case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
538     Opcode = Mips::OR;
539     break;
540   case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
541     Opcode = Mips::XOR;
542     break;
543   case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
544     IsNand = true;
545     AND = Mips::AND;
546     NOR = Mips::NOR;
547     break;
548   case Mips::ATOMIC_SWAP_I32_POSTRA:
549     OR = Mips::OR;
550     break;
551   case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
552     Opcode = Mips::DADDu;
553     break;
554   case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
555     Opcode = Mips::DSUBu;
556     break;
557   case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
558     Opcode = Mips::AND64;
559     break;
560   case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
561     Opcode = Mips::OR64;
562     break;
563   case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
564     Opcode = Mips::XOR64;
565     break;
566   case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
567     IsNand = true;
568     AND = Mips::AND64;
569     NOR = Mips::NOR64;
570     break;
571   case Mips::ATOMIC_SWAP_I64_POSTRA:
572     OR = Mips::OR64;
573     break;
574   default:
575     llvm_unreachable("Unknown pseudo atomic!");
576   }
577 
578   const BasicBlock *LLVM_BB = BB.getBasicBlock();
579   MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
580   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
581   MachineFunction::iterator It = ++BB.getIterator();
582   MF->insert(It, loopMBB);
583   MF->insert(It, exitMBB);
584 
585   exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
586   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
587 
588   BB.addSuccessor(loopMBB, BranchProbability::getOne());
589   loopMBB->addSuccessor(exitMBB);
590   loopMBB->addSuccessor(loopMBB);
591   loopMBB->normalizeSuccProbs();
592 
593   BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
594   assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!");
595   assert((OldVal != Incr) && "Clobbered the wrong reg!");
596   if (Opcode) {
597     BuildMI(loopMBB, DL, TII->get(Opcode), Scratch).addReg(OldVal).addReg(Incr);
598   } else if (IsNand) {
599     assert(AND && NOR &&
600            "Unknown nand instruction for atomic pseudo expansion");
601     BuildMI(loopMBB, DL, TII->get(AND), Scratch).addReg(OldVal).addReg(Incr);
602     BuildMI(loopMBB, DL, TII->get(NOR), Scratch).addReg(ZERO).addReg(Scratch);
603   } else {
604     assert(OR && "Unknown instruction for atomic pseudo expansion!");
605     BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO);
606   }
607 
608   BuildMI(loopMBB, DL, TII->get(SC), Scratch).addReg(Scratch).addReg(Ptr).addImm(0);
609   BuildMI(loopMBB, DL, TII->get(BEQ)).addReg(Scratch).addReg(ZERO).addMBB(loopMBB);
610 
611   NMBBI = BB.end();
612   I->eraseFromParent();
613 
614   LivePhysRegs LiveRegs;
615   computeAndAddLiveIns(LiveRegs, *loopMBB);
616   computeAndAddLiveIns(LiveRegs, *exitMBB);
617 
618   return true;
619 }
620 
621 bool MipsExpandPseudo::expandMI(MachineBasicBlock &MBB,
622                                 MachineBasicBlock::iterator MBBI,
623                                 MachineBasicBlock::iterator &NMBB) {
624 
625   bool Modified = false;
626 
627   switch (MBBI->getOpcode()) {
628   case Mips::ATOMIC_CMP_SWAP_I32_POSTRA:
629   case Mips::ATOMIC_CMP_SWAP_I64_POSTRA:
630     return expandAtomicCmpSwap(MBB, MBBI, NMBB);
631   case Mips::ATOMIC_CMP_SWAP_I8_POSTRA:
632   case Mips::ATOMIC_CMP_SWAP_I16_POSTRA:
633     return expandAtomicCmpSwapSubword(MBB, MBBI, NMBB);
634   case Mips::ATOMIC_SWAP_I8_POSTRA:
635   case Mips::ATOMIC_SWAP_I16_POSTRA:
636   case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
637   case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
638   case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
639   case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
640   case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
641   case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
642   case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
643   case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
644   case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
645   case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
646   case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
647   case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
648     return expandAtomicBinOpSubword(MBB, MBBI, NMBB);
649   case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
650   case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
651   case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
652   case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
653   case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
654   case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
655   case Mips::ATOMIC_SWAP_I32_POSTRA:
656     return expandAtomicBinOp(MBB, MBBI, NMBB, 4);
657   case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
658   case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
659   case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
660   case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
661   case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
662   case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
663   case Mips::ATOMIC_SWAP_I64_POSTRA:
664     return expandAtomicBinOp(MBB, MBBI, NMBB, 8);
665   default:
666     return Modified;
667   }
668 }
669 
670 bool MipsExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
671   bool Modified = false;
672 
673   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
674   while (MBBI != E) {
675     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
676     Modified |= expandMI(MBB, MBBI, NMBBI);
677     MBBI = NMBBI;
678   }
679 
680   return Modified;
681 }
682 
683 bool MipsExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
684   STI = &static_cast<const MipsSubtarget &>(MF.getSubtarget());
685   TII = STI->getInstrInfo();
686 
687   bool Modified = false;
688   for (MachineFunction::iterator MFI = MF.begin(), E = MF.end(); MFI != E;
689        ++MFI)
690     Modified |= expandMBB(*MFI);
691 
692   if (Modified)
693     MF.RenumberBlocks();
694 
695   return Modified;
696 }
697 
698 /// createMipsExpandPseudoPass - returns an instance of the pseudo instruction
699 /// expansion pass.
700 FunctionPass *llvm::createMipsExpandPseudoPass() {
701   return new MipsExpandPseudo();
702 }
703