1 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 /// \file This file contains the AArch64 implementation of the DAG scheduling 11 /// mutation to pair instructions back to back. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "AArch64Subtarget.h" 16 #include "llvm/CodeGen/MacroFusion.h" 17 #include "llvm/CodeGen/TargetInstrInfo.h" 18 19 using namespace llvm; 20 21 namespace { 22 23 /// CMN, CMP, TST followed by Bcc 24 static bool isArithmeticBccPair(const MachineInstr *FirstMI, 25 const MachineInstr &SecondMI) { 26 if (SecondMI.getOpcode() != AArch64::Bcc) 27 return false; 28 29 // Assume the 1st instr to be a wildcard if it is unspecified. 30 if (FirstMI == nullptr) 31 return true; 32 33 switch (FirstMI->getOpcode()) { 34 case AArch64::ADDSWri: 35 case AArch64::ADDSWrr: 36 case AArch64::ADDSXri: 37 case AArch64::ADDSXrr: 38 case AArch64::ANDSWri: 39 case AArch64::ANDSWrr: 40 case AArch64::ANDSXri: 41 case AArch64::ANDSXrr: 42 case AArch64::SUBSWri: 43 case AArch64::SUBSWrr: 44 case AArch64::SUBSXri: 45 case AArch64::SUBSXrr: 46 case AArch64::BICSWrr: 47 case AArch64::BICSXrr: 48 return true; 49 case AArch64::ADDSWrs: 50 case AArch64::ADDSXrs: 51 case AArch64::ANDSWrs: 52 case AArch64::ANDSXrs: 53 case AArch64::SUBSWrs: 54 case AArch64::SUBSXrs: 55 case AArch64::BICSWrs: 56 case AArch64::BICSXrs: 57 // Shift value can be 0 making these behave like the "rr" variant... 58 return !AArch64InstrInfo::hasShiftedReg(*FirstMI); 59 } 60 61 return false; 62 } 63 64 /// ALU operations followed by CBZ/CBNZ. 65 static bool isArithmeticCbzPair(const MachineInstr *FirstMI, 66 const MachineInstr &SecondMI) { 67 if (SecondMI.getOpcode() != AArch64::CBZW && 68 SecondMI.getOpcode() != AArch64::CBZX && 69 SecondMI.getOpcode() != AArch64::CBNZW && 70 SecondMI.getOpcode() != AArch64::CBNZX) 71 return false; 72 73 // Assume the 1st instr to be a wildcard if it is unspecified. 74 if (FirstMI == nullptr) 75 return true; 76 77 switch (FirstMI->getOpcode()) { 78 case AArch64::ADDWri: 79 case AArch64::ADDWrr: 80 case AArch64::ADDXri: 81 case AArch64::ADDXrr: 82 case AArch64::ANDWri: 83 case AArch64::ANDWrr: 84 case AArch64::ANDXri: 85 case AArch64::ANDXrr: 86 case AArch64::EORWri: 87 case AArch64::EORWrr: 88 case AArch64::EORXri: 89 case AArch64::EORXrr: 90 case AArch64::ORRWri: 91 case AArch64::ORRWrr: 92 case AArch64::ORRXri: 93 case AArch64::ORRXrr: 94 case AArch64::SUBWri: 95 case AArch64::SUBWrr: 96 case AArch64::SUBXri: 97 case AArch64::SUBXrr: 98 return true; 99 case AArch64::ADDWrs: 100 case AArch64::ADDXrs: 101 case AArch64::ANDWrs: 102 case AArch64::ANDXrs: 103 case AArch64::SUBWrs: 104 case AArch64::SUBXrs: 105 case AArch64::BICWrs: 106 case AArch64::BICXrs: 107 // Shift value can be 0 making these behave like the "rr" variant... 108 return !AArch64InstrInfo::hasShiftedReg(*FirstMI); 109 } 110 111 return false; 112 } 113 114 /// AES crypto encoding or decoding. 115 static bool isAESPair(const MachineInstr *FirstMI, 116 const MachineInstr &SecondMI) { 117 // Assume the 1st instr to be a wildcard if it is unspecified. 118 switch (SecondMI.getOpcode()) { 119 // AES encode. 120 case AArch64::AESMCrr: 121 case AArch64::AESMCrrTied: 122 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr; 123 // AES decode. 124 case AArch64::AESIMCrr: 125 case AArch64::AESIMCrrTied: 126 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr; 127 } 128 129 return false; 130 } 131 132 /// AESE/AESD/PMULL + EOR. 133 static bool isCryptoEORPair(const MachineInstr *FirstMI, 134 const MachineInstr &SecondMI) { 135 if (SecondMI.getOpcode() != AArch64::EORv16i8) 136 return false; 137 138 // Assume the 1st instr to be a wildcard if it is unspecified. 139 if (FirstMI == nullptr) 140 return true; 141 142 switch (FirstMI->getOpcode()) { 143 case AArch64::AESErr: 144 case AArch64::AESDrr: 145 case AArch64::PMULLv16i8: 146 case AArch64::PMULLv8i8: 147 case AArch64::PMULLv1i64: 148 case AArch64::PMULLv2i64: 149 return true; 150 } 151 152 return false; 153 } 154 155 /// Literal generation. 156 static bool isLiteralsPair(const MachineInstr *FirstMI, 157 const MachineInstr &SecondMI) { 158 // Assume the 1st instr to be a wildcard if it is unspecified. 159 160 // PC relative address. 161 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) && 162 SecondMI.getOpcode() == AArch64::ADDXri) 163 return true; 164 165 // 32 bit immediate. 166 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) && 167 (SecondMI.getOpcode() == AArch64::MOVKWi && 168 SecondMI.getOperand(3).getImm() == 16)) 169 return true; 170 171 // Lower half of 64 bit immediate. 172 if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) && 173 (SecondMI.getOpcode() == AArch64::MOVKXi && 174 SecondMI.getOperand(3).getImm() == 16)) 175 return true; 176 177 // Upper half of 64 bit immediate. 178 if ((FirstMI == nullptr || 179 (FirstMI->getOpcode() == AArch64::MOVKXi && 180 FirstMI->getOperand(3).getImm() == 32)) && 181 (SecondMI.getOpcode() == AArch64::MOVKXi && 182 SecondMI.getOperand(3).getImm() == 48)) 183 return true; 184 185 return false; 186 } 187 188 /// Fuse address generation and loads or stores. 189 static bool isAddressLdStPair(const MachineInstr *FirstMI, 190 const MachineInstr &SecondMI) { 191 switch (SecondMI.getOpcode()) { 192 case AArch64::STRBBui: 193 case AArch64::STRBui: 194 case AArch64::STRDui: 195 case AArch64::STRHHui: 196 case AArch64::STRHui: 197 case AArch64::STRQui: 198 case AArch64::STRSui: 199 case AArch64::STRWui: 200 case AArch64::STRXui: 201 case AArch64::LDRBBui: 202 case AArch64::LDRBui: 203 case AArch64::LDRDui: 204 case AArch64::LDRHHui: 205 case AArch64::LDRHui: 206 case AArch64::LDRQui: 207 case AArch64::LDRSui: 208 case AArch64::LDRWui: 209 case AArch64::LDRXui: 210 case AArch64::LDRSBWui: 211 case AArch64::LDRSBXui: 212 case AArch64::LDRSHWui: 213 case AArch64::LDRSHXui: 214 case AArch64::LDRSWui: 215 // Assume the 1st instr to be a wildcard if it is unspecified. 216 if (FirstMI == nullptr) 217 return true; 218 219 switch (FirstMI->getOpcode()) { 220 case AArch64::ADR: 221 return SecondMI.getOperand(2).getImm() == 0; 222 case AArch64::ADRP: 223 return true; 224 } 225 } 226 227 return false; 228 } 229 230 /// Compare and conditional select. 231 static bool isCCSelectPair(const MachineInstr *FirstMI, 232 const MachineInstr &SecondMI) { 233 // 32 bits 234 if (SecondMI.getOpcode() == AArch64::CSELWr) { 235 // Assume the 1st instr to be a wildcard if it is unspecified. 236 if (FirstMI == nullptr) 237 return true; 238 239 if (FirstMI->definesRegister(AArch64::WZR)) 240 switch (FirstMI->getOpcode()) { 241 case AArch64::SUBSWrs: 242 return !AArch64InstrInfo::hasShiftedReg(*FirstMI); 243 case AArch64::SUBSWrx: 244 return !AArch64InstrInfo::hasExtendedReg(*FirstMI); 245 case AArch64::SUBSWrr: 246 case AArch64::SUBSWri: 247 return true; 248 } 249 } 250 251 // 64 bits 252 if (SecondMI.getOpcode() == AArch64::CSELXr) { 253 // Assume the 1st instr to be a wildcard if it is unspecified. 254 if (FirstMI == nullptr) 255 return true; 256 257 if (FirstMI->definesRegister(AArch64::XZR)) 258 switch (FirstMI->getOpcode()) { 259 case AArch64::SUBSXrs: 260 return !AArch64InstrInfo::hasShiftedReg(*FirstMI); 261 case AArch64::SUBSXrx: 262 case AArch64::SUBSXrx64: 263 return !AArch64InstrInfo::hasExtendedReg(*FirstMI); 264 case AArch64::SUBSXrr: 265 case AArch64::SUBSXri: 266 return true; 267 } 268 } 269 270 return false; 271 } 272 273 /// Check if the instr pair, FirstMI and SecondMI, should be fused 274 /// together. Given SecondMI, when FirstMI is unspecified, then check if 275 /// SecondMI may be part of a fused pair at all. 276 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII, 277 const TargetSubtargetInfo &TSI, 278 const MachineInstr *FirstMI, 279 const MachineInstr &SecondMI) { 280 const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI); 281 282 // All checking functions assume that the 1st instr is a wildcard if it is 283 // unspecified. 284 if (ST.hasArithmeticBccFusion() && isArithmeticBccPair(FirstMI, SecondMI)) 285 return true; 286 if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI)) 287 return true; 288 if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI)) 289 return true; 290 if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI)) 291 return true; 292 if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI)) 293 return true; 294 if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI)) 295 return true; 296 if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI)) 297 return true; 298 299 return false; 300 } 301 302 } // end namespace 303 304 305 namespace llvm { 306 307 std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation () { 308 return createMacroFusionDAGMutation(shouldScheduleAdjacent); 309 } 310 311 } // end namespace llvm 312