1// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9/// 10/// \file 11/// WebAssembly Atomic operand code-gen constructs. 12/// 13//===----------------------------------------------------------------------===// 14 15//===----------------------------------------------------------------------===// 16// Atomic loads 17//===----------------------------------------------------------------------===// 18 19multiclass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 20 list<dag> pattern_r, string asmstr_r = "", 21 string asmstr_s = "", bits<32> inst = -1> { 22 defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 23 inst>, 24 Requires<[HasAtomics]>; 25} 26 27let Defs = [ARGUMENTS] in { 28defm ATOMIC_LOAD_I32 : WebAssemblyLoad<I32, "i32.atomic.load", 0xfe10>; 29defm ATOMIC_LOAD_I64 : WebAssemblyLoad<I64, "i64.atomic.load", 0xfe11>; 30} // Defs = [ARGUMENTS] 31 32// Select loads with no constant offset. 33let Predicates = [HasAtomics] in { 34def : LoadPatNoOffset<i32, atomic_load_32, ATOMIC_LOAD_I32>; 35def : LoadPatNoOffset<i64, atomic_load_64, ATOMIC_LOAD_I64>; 36 37// Select loads with a constant offset. 38 39// Pattern with address + immediate offset 40def : LoadPatImmOff<i32, atomic_load_32, regPlusImm, ATOMIC_LOAD_I32>; 41def : LoadPatImmOff<i64, atomic_load_64, regPlusImm, ATOMIC_LOAD_I64>; 42def : LoadPatImmOff<i32, atomic_load_32, or_is_add, ATOMIC_LOAD_I32>; 43def : LoadPatImmOff<i64, atomic_load_64, or_is_add, ATOMIC_LOAD_I64>; 44 45def : LoadPatGlobalAddr<i32, atomic_load_32, ATOMIC_LOAD_I32>; 46def : LoadPatGlobalAddr<i64, atomic_load_64, ATOMIC_LOAD_I64>; 47 48def : LoadPatExternalSym<i32, atomic_load_32, ATOMIC_LOAD_I32>; 49def : LoadPatExternalSym<i64, atomic_load_64, ATOMIC_LOAD_I64>; 50 51// Select loads with just a constant offset. 52def : LoadPatOffsetOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 53def : LoadPatOffsetOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 54 55def : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 56def : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 57 58def : LoadPatExternSymOffOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 59def : LoadPatExternSymOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 60 61} // Predicates = [HasAtomics] 62 63// Extending loads. Note that there are only zero-extending atomic loads, no 64// sign-extending loads. 65let Defs = [ARGUMENTS] in { 66defm ATOMIC_LOAD8_U_I32 : WebAssemblyLoad<I32, "i32.atomic.load8_u", 0xfe12>; 67defm ATOMIC_LOAD16_U_I32 : WebAssemblyLoad<I32, "i32.atomic.load16_u", 0xfe13>; 68defm ATOMIC_LOAD8_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load8_u", 0xfe14>; 69defm ATOMIC_LOAD16_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load16_u", 0xfe15>; 70defm ATOMIC_LOAD32_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load32_u", 0xfe16>; 71} // Defs = [ARGUMENTS] 72 73// Fragments for extending loads. These are different from regular loads because 74// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 75// therefore don't have the extension type field. So instead of matching that, 76// we match the patterns that the type legalizer expands them to. 77 78// We directly match zext patterns and select the zext atomic loads. 79// i32 (zext (i8 (atomic_load_8))) gets legalized to 80// i32 (and (i32 (atomic_load_8)), 255) 81// These can be selected to a single zero-extending atomic load instruction. 82def zext_aload_8_32 : 83 PatFrag<(ops node:$addr), (and (i32 (atomic_load_8 node:$addr)), 255)>; 84def zext_aload_16_32 : 85 PatFrag<(ops node:$addr), (and (i32 (atomic_load_16 node:$addr)), 65535)>; 86// Unlike regular loads, extension to i64 is handled differently than i32. 87// i64 (zext (i8 (atomic_load_8))) gets legalized to 88// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 89def zext_aload_8_64 : 90 PatFrag<(ops node:$addr), 91 (and (i64 (anyext (i32 (atomic_load_8 node:$addr)))), 255)>; 92def zext_aload_16_64 : 93 PatFrag<(ops node:$addr), 94 (and (i64 (anyext (i32 (atomic_load_16 node:$addr)))), 65535)>; 95def zext_aload_32_64 : 96 PatFrag<(ops node:$addr), 97 (zext (i32 (atomic_load node:$addr)))>; 98 99// We don't have single sext atomic load instructions. So for sext loads, we 100// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 101// results) and select a zext load; the next instruction will be sext_inreg 102// which is selected by itself. 103def sext_aload_8_64 : 104 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 105def sext_aload_16_64 : 106 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 107 108let Predicates = [HasAtomics] in { 109// Select zero-extending loads with no constant offset. 110def : LoadPatNoOffset<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 111def : LoadPatNoOffset<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 112def : LoadPatNoOffset<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 113def : LoadPatNoOffset<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 114def : LoadPatNoOffset<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 115 116// Select sign-extending loads with no constant offset 117def : LoadPatNoOffset<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 118def : LoadPatNoOffset<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 119def : LoadPatNoOffset<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 120def : LoadPatNoOffset<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 121// 32->64 sext load gets selected as i32.atomic.load, i64.extend_s/i32 122 123// Zero-extending loads with constant offset 124def : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, ATOMIC_LOAD8_U_I32>; 125def : LoadPatImmOff<i32, zext_aload_16_32, regPlusImm, ATOMIC_LOAD16_U_I32>; 126def : LoadPatImmOff<i32, zext_aload_8_32, or_is_add, ATOMIC_LOAD8_U_I32>; 127def : LoadPatImmOff<i32, zext_aload_16_32, or_is_add, ATOMIC_LOAD16_U_I32>; 128def : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 129def : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 130def : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, ATOMIC_LOAD32_U_I64>; 131def : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 132def : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 133def : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, ATOMIC_LOAD32_U_I64>; 134 135// Sign-extending loads with constant offset 136def : LoadPatImmOff<i32, atomic_load_8, regPlusImm, ATOMIC_LOAD8_U_I32>; 137def : LoadPatImmOff<i32, atomic_load_16, regPlusImm, ATOMIC_LOAD16_U_I32>; 138def : LoadPatImmOff<i32, atomic_load_8, or_is_add, ATOMIC_LOAD8_U_I32>; 139def : LoadPatImmOff<i32, atomic_load_16, or_is_add, ATOMIC_LOAD16_U_I32>; 140def : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 141def : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 142def : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 143def : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 144// No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 145 146def : LoadPatGlobalAddr<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 147def : LoadPatGlobalAddr<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 148def : LoadPatGlobalAddr<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 149def : LoadPatGlobalAddr<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 150def : LoadPatGlobalAddr<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 151def : LoadPatGlobalAddr<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 152def : LoadPatGlobalAddr<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 153def : LoadPatGlobalAddr<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 154def : LoadPatGlobalAddr<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 155 156def : LoadPatExternalSym<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 157def : LoadPatExternalSym<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 158def : LoadPatExternalSym<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 159def : LoadPatExternalSym<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 160def : LoadPatExternalSym<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 161def : LoadPatExternalSym<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 162def : LoadPatExternalSym<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 163def : LoadPatExternalSym<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 164def : LoadPatExternalSym<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 165 166// Extending loads with just a constant offset 167def : LoadPatOffsetOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 168def : LoadPatOffsetOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 169def : LoadPatOffsetOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 170def : LoadPatOffsetOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 171def : LoadPatOffsetOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 172def : LoadPatOffsetOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 173def : LoadPatOffsetOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 174def : LoadPatOffsetOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 175def : LoadPatOffsetOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 176 177def : LoadPatGlobalAddrOffOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 178def : LoadPatGlobalAddrOffOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 179def : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 180def : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 181def : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 182def : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 183def : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 184def : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 185def : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 186 187def : LoadPatExternSymOffOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 188def : LoadPatExternSymOffOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 189def : LoadPatExternSymOffOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 190def : LoadPatExternSymOffOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 191def : LoadPatExternSymOffOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 192def : LoadPatExternSymOffOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 193def : LoadPatExternSymOffOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 194def : LoadPatExternSymOffOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 195def : LoadPatExternSymOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 196 197} // Predicates = [HasAtomics] 198 199//===----------------------------------------------------------------------===// 200// Atomic stores 201//===----------------------------------------------------------------------===// 202 203let Defs = [ARGUMENTS] in { 204defm ATOMIC_STORE_I32 : WebAssemblyStore<I32, "i32.atomic.store", 0xfe17>; 205defm ATOMIC_STORE_I64 : WebAssemblyStore<I64, "i64.atomic.store", 0xfe18>; 206} // Defs = [ARGUMENTS] 207 208// We need an 'atomic' version of store patterns because store and atomic_store 209// nodes have different operand orders: 210// store: (store $val, $ptr) 211// atomic_store: (store $ptr, $val) 212 213let Predicates = [HasAtomics] in { 214 215// Select stores with no constant offset. 216class AStorePatNoOffset<ValueType ty, PatFrag kind, NI inst> : 217 Pat<(kind I32:$addr, ty:$val), (inst 0, 0, I32:$addr, ty:$val)>; 218def : AStorePatNoOffset<i32, atomic_store_32, ATOMIC_STORE_I32>; 219def : AStorePatNoOffset<i64, atomic_store_64, ATOMIC_STORE_I64>; 220 221// Select stores with a constant offset. 222 223// Pattern with address + immediate offset 224class AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 225 Pat<(kind (operand I32:$addr, imm:$off), ty:$val), 226 (inst 0, imm:$off, I32:$addr, ty:$val)>; 227def : AStorePatImmOff<i32, atomic_store_32, regPlusImm, ATOMIC_STORE_I32>; 228def : AStorePatImmOff<i64, atomic_store_64, regPlusImm, ATOMIC_STORE_I64>; 229def : AStorePatImmOff<i32, atomic_store_32, or_is_add, ATOMIC_STORE_I32>; 230def : AStorePatImmOff<i64, atomic_store_64, or_is_add, ATOMIC_STORE_I64>; 231 232class AStorePatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 233 Pat<(kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 234 ty:$val), 235 (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>; 236def : AStorePatGlobalAddr<i32, atomic_store_32, ATOMIC_STORE_I32>; 237def : AStorePatGlobalAddr<i64, atomic_store_64, ATOMIC_STORE_I64>; 238 239class AStorePatExternalSym<ValueType ty, PatFrag kind, NI inst> : 240 Pat<(kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), ty:$val), 241 (inst 0, texternalsym:$off, I32:$addr, ty:$val)>; 242def : AStorePatExternalSym<i32, atomic_store_32, ATOMIC_STORE_I32>; 243def : AStorePatExternalSym<i64, atomic_store_64, ATOMIC_STORE_I64>; 244 245// Select stores with just a constant offset. 246class AStorePatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 247 Pat<(kind imm:$off, ty:$val), (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 248def : AStorePatOffsetOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 249def : AStorePatOffsetOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 250 251class AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 252 Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 253 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 254def : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 255def : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 256 257class AStorePatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> : 258 Pat<(kind (WebAssemblywrapper texternalsym:$off), ty:$val), 259 (inst 0, texternalsym:$off, (CONST_I32 0), ty:$val)>; 260def : AStorePatExternSymOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 261def : AStorePatExternSymOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 262 263} // Predicates = [HasAtomics] 264 265// Truncating stores. 266let Defs = [ARGUMENTS] in { 267defm ATOMIC_STORE8_I32 : WebAssemblyStore<I32, "i32.atomic.store8", 0xfe19>; 268defm ATOMIC_STORE16_I32 : WebAssemblyStore<I32, "i32.atomic.store16", 0xfe1a>; 269defm ATOMIC_STORE8_I64 : WebAssemblyStore<I64, "i64.atomic.store8", 0xfe1b>; 270defm ATOMIC_STORE16_I64 : WebAssemblyStore<I64, "i64.atomic.store16", 0xfe1c>; 271defm ATOMIC_STORE32_I64 : WebAssemblyStore<I64, "i64.atomic.store32", 0xfe1d>; 272} // Defs = [ARGUMENTS] 273 274// Fragments for truncating stores. 275 276// We don't have single truncating atomic store instructions. For 32-bit 277// instructions, we just need to match bare atomic stores. On the other hand, 278// truncating stores from i64 values are once truncated to i32 first. 279class trunc_astore_64<PatFrag kind> : 280 PatFrag<(ops node:$addr, node:$val), 281 (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 282def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 283def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 284def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 285 286let Predicates = [HasAtomics] in { 287 288// Truncating stores with no constant offset 289def : AStorePatNoOffset<i32, atomic_store_8, ATOMIC_STORE8_I32>; 290def : AStorePatNoOffset<i32, atomic_store_16, ATOMIC_STORE16_I32>; 291def : AStorePatNoOffset<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 292def : AStorePatNoOffset<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 293def : AStorePatNoOffset<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 294 295// Truncating stores with a constant offset 296def : AStorePatImmOff<i32, atomic_store_8, regPlusImm, ATOMIC_STORE8_I32>; 297def : AStorePatImmOff<i32, atomic_store_16, regPlusImm, ATOMIC_STORE16_I32>; 298def : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, ATOMIC_STORE8_I64>; 299def : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, ATOMIC_STORE16_I64>; 300def : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, ATOMIC_STORE32_I64>; 301def : AStorePatImmOff<i32, atomic_store_8, or_is_add, ATOMIC_STORE8_I32>; 302def : AStorePatImmOff<i32, atomic_store_16, or_is_add, ATOMIC_STORE16_I32>; 303def : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, ATOMIC_STORE8_I64>; 304def : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, ATOMIC_STORE16_I64>; 305def : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, ATOMIC_STORE32_I64>; 306 307def : AStorePatGlobalAddr<i32, atomic_store_8, ATOMIC_STORE8_I32>; 308def : AStorePatGlobalAddr<i32, atomic_store_16, ATOMIC_STORE16_I32>; 309def : AStorePatGlobalAddr<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 310def : AStorePatGlobalAddr<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 311def : AStorePatGlobalAddr<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 312 313def : AStorePatExternalSym<i32, atomic_store_8, ATOMIC_STORE8_I32>; 314def : AStorePatExternalSym<i32, atomic_store_16, ATOMIC_STORE16_I32>; 315def : AStorePatExternalSym<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 316def : AStorePatExternalSym<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 317def : AStorePatExternalSym<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 318 319// Truncating stores with just a constant offset 320def : AStorePatOffsetOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 321def : AStorePatOffsetOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 322def : AStorePatOffsetOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 323def : AStorePatOffsetOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 324def : AStorePatOffsetOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 325 326def : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 327def : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 328def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 329def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 330def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 331 332def : AStorePatExternSymOffOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 333def : AStorePatExternSymOffOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 334def : AStorePatExternSymOffOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 335def : AStorePatExternSymOffOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 336def : AStorePatExternSymOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 337 338} // Predicates = [HasAtomics] 339 340//===----------------------------------------------------------------------===// 341// Atomic binary read-modify-writes 342//===----------------------------------------------------------------------===// 343 344let Defs = [ARGUMENTS] in { 345 346multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string Name, int Opcode> { 347 defm "" : I<(outs rc:$dst), 348 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 349 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 350 !strconcat(Name, "\t$dst, ${off}(${addr})${p2align}, $val"), 351 !strconcat(Name, "\t${off}, ${p2align}"), Opcode>; 352} 353 354defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0xfe1e>; 355defm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0xfe1f>; 356defm ATOMIC_RMW8_U_ADD_I32 : 357 WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.add", 0xfe20>; 358defm ATOMIC_RMW16_U_ADD_I32 : 359 WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.add", 0xfe21>; 360defm ATOMIC_RMW8_U_ADD_I64 : 361 WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.add", 0xfe22>; 362defm ATOMIC_RMW16_U_ADD_I64 : 363 WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.add", 0xfe23>; 364defm ATOMIC_RMW32_U_ADD_I64 : 365 WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.add", 0xfe24>; 366 367defm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0xfe25>; 368defm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0xfe26>; 369defm ATOMIC_RMW8_U_SUB_I32 : 370 WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.sub", 0xfe27>; 371defm ATOMIC_RMW16_U_SUB_I32 : 372 WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.sub", 0xfe28>; 373defm ATOMIC_RMW8_U_SUB_I64 : 374 WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.sub", 0xfe29>; 375defm ATOMIC_RMW16_U_SUB_I64 : 376 WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.sub", 0xfe2a>; 377defm ATOMIC_RMW32_U_SUB_I64 : 378 WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.sub", 0xfe2b>; 379 380defm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0xfe2c>; 381defm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0xfe2d>; 382defm ATOMIC_RMW8_U_AND_I32 : 383 WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.and", 0xfe2e>; 384defm ATOMIC_RMW16_U_AND_I32 : 385 WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.and", 0xfe2f>; 386defm ATOMIC_RMW8_U_AND_I64 : 387 WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.and", 0xfe30>; 388defm ATOMIC_RMW16_U_AND_I64 : 389 WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.and", 0xfe31>; 390defm ATOMIC_RMW32_U_AND_I64 : 391 WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.and", 0xfe32>; 392 393defm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0xfe33>; 394defm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0xfe34>; 395defm ATOMIC_RMW8_U_OR_I32 : 396 WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.or", 0xfe35>; 397defm ATOMIC_RMW16_U_OR_I32 : 398 WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.or", 0xfe36>; 399defm ATOMIC_RMW8_U_OR_I64 : 400 WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.or", 0xfe37>; 401defm ATOMIC_RMW16_U_OR_I64 : 402 WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.or", 0xfe38>; 403defm ATOMIC_RMW32_U_OR_I64 : 404 WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.or", 0xfe39>; 405 406defm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0xfe3a>; 407defm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0xfe3b>; 408defm ATOMIC_RMW8_U_XOR_I32 : 409 WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.xor", 0xfe3c>; 410defm ATOMIC_RMW16_U_XOR_I32 : 411 WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.xor", 0xfe3d>; 412defm ATOMIC_RMW8_U_XOR_I64 : 413 WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.xor", 0xfe3e>; 414defm ATOMIC_RMW16_U_XOR_I64 : 415 WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.xor", 0xfe3f>; 416defm ATOMIC_RMW32_U_XOR_I64 : 417 WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.xor", 0xfe40>; 418 419defm ATOMIC_RMW_XCHG_I32 : 420 WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0xfe41>; 421defm ATOMIC_RMW_XCHG_I64 : 422 WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0xfe42>; 423defm ATOMIC_RMW8_U_XCHG_I32 : 424 WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.xchg", 0xfe43>; 425defm ATOMIC_RMW16_U_XCHG_I32 : 426 WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.xchg", 0xfe44>; 427defm ATOMIC_RMW8_U_XCHG_I64 : 428 WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.xchg", 0xfe45>; 429defm ATOMIC_RMW16_U_XCHG_I64 : 430 WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.xchg", 0xfe46>; 431defm ATOMIC_RMW32_U_XCHG_I64 : 432 WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.xchg", 0xfe47>; 433} 434 435// Select binary RMWs with no constant offset. 436class BinRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 437 Pat<(ty (kind I32:$addr, ty:$val)), (inst 0, 0, I32:$addr, ty:$val)>; 438 439// Select binary RMWs with a constant offset. 440 441// Pattern with address + immediate offset 442class BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 443 Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), 444 (inst 0, imm:$off, I32:$addr, ty:$val)>; 445 446class BinRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 447 Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 448 ty:$val)), 449 (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>; 450 451class BinRMWPatExternalSym<ValueType ty, PatFrag kind, NI inst> : 452 Pat<(ty (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), 453 ty:$val)), 454 (inst 0, texternalsym:$off, I32:$addr, ty:$val)>; 455 456// Select binary RMWs with just a constant offset. 457class BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 458 Pat<(ty (kind imm:$off, ty:$val)), 459 (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 460 461class BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 462 Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 463 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 464 465class BinRMWPatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> : 466 Pat<(ty (kind (WebAssemblywrapper texternalsym:$off), ty:$val)), 467 (inst 0, texternalsym:$off, (CONST_I32 0), ty:$val)>; 468 469// Patterns for various addressing modes. 470multiclass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 471 NI inst_64> { 472 def : BinRMWPatNoOffset<i32, rmw_32, inst_32>; 473 def : BinRMWPatNoOffset<i64, rmw_64, inst_64>; 474 475 def : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 476 def : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 477 def : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 478 def : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 479 480 def : BinRMWPatGlobalAddr<i32, rmw_32, inst_32>; 481 def : BinRMWPatGlobalAddr<i64, rmw_64, inst_64>; 482 483 def : BinRMWPatExternalSym<i32, rmw_32, inst_32>; 484 def : BinRMWPatExternalSym<i64, rmw_64, inst_64>; 485 486 def : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>; 487 def : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>; 488 489 def : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 490 def : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 491 492 def : BinRMWPatExternSymOffOnly<i32, rmw_32, inst_32>; 493 def : BinRMWPatExternSymOffOnly<i64, rmw_64, inst_64>; 494} 495 496let Predicates = [HasAtomics] in { 497defm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, ATOMIC_RMW_ADD_I32, 498 ATOMIC_RMW_ADD_I64>; 499defm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, ATOMIC_RMW_SUB_I32, 500 ATOMIC_RMW_SUB_I64>; 501defm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, ATOMIC_RMW_AND_I32, 502 ATOMIC_RMW_AND_I64>; 503defm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, ATOMIC_RMW_OR_I32, 504 ATOMIC_RMW_OR_I64>; 505defm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, ATOMIC_RMW_XOR_I32, 506 ATOMIC_RMW_XOR_I64>; 507defm : BinRMWPattern<atomic_swap_32, atomic_swap_64, ATOMIC_RMW_XCHG_I32, 508 ATOMIC_RMW_XCHG_I64>; 509} // Predicates = [HasAtomics] 510 511// Truncating & zero-extending binary RMW patterns. 512// These are combined patterns of truncating store patterns and zero-extending 513// load patterns above. 514class zext_bin_rmw_8_32<PatFrag kind> : 515 PatFrag<(ops node:$addr, node:$val), 516 (and (i32 (kind node:$addr, node:$val)), 255)>; 517class zext_bin_rmw_16_32<PatFrag kind> : 518 PatFrag<(ops node:$addr, node:$val), 519 (and (i32 (kind node:$addr, node:$val)), 65535)>; 520class zext_bin_rmw_8_64<PatFrag kind> : 521 PatFrag<(ops node:$addr, node:$val), 522 (and (i64 (anyext (i32 (kind node:$addr, 523 (i32 (trunc (i64 node:$val))))))), 255)>; 524class zext_bin_rmw_16_64<PatFrag kind> : 525 PatFrag<(ops node:$addr, node:$val), 526 (and (i64 (anyext (i32 (kind node:$addr, 527 (i32 (trunc (i64 node:$val))))))), 65535)>; 528class zext_bin_rmw_32_64<PatFrag kind> : 529 PatFrag<(ops node:$addr, node:$val), 530 (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 531 532// Truncating & sign-extending binary RMW patterns. 533// These are combined patterns of truncating store patterns and sign-extending 534// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 535// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 536// is selected by itself. 537class sext_bin_rmw_8_32<PatFrag kind> : 538 PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 539class sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 540class sext_bin_rmw_8_64<PatFrag kind> : 541 PatFrag<(ops node:$addr, node:$val), 542 (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 543class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 544// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_s/i32 545 546// Patterns for various addressing modes for truncating-extending binary RMWs. 547multiclass BinRMWTruncExtPattern< 548 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 549 NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 550 // Truncating-extending binary RMWs with no constant offset 551 def : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 552 def : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 553 def : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 554 def : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 555 def : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 556 557 def : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 558 def : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 559 def : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 560 def : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 561 562 // Truncating-extending binary RMWs with a constant offset 563 def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 564 def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 565 def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 566 def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 567 def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 568 def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 569 def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 570 def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 571 def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 572 def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 573 574 def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 575 def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 576 def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 577 def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 578 def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 579 def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 580 def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 581 def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 582 583 def : BinRMWPatGlobalAddr<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 584 def : BinRMWPatGlobalAddr<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 585 def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 586 def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 587 def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 588 589 def : BinRMWPatGlobalAddr<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 590 def : BinRMWPatGlobalAddr<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 591 def : BinRMWPatGlobalAddr<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 592 def : BinRMWPatGlobalAddr<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 593 594 def : BinRMWPatExternalSym<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 595 def : BinRMWPatExternalSym<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 596 def : BinRMWPatExternalSym<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 597 def : BinRMWPatExternalSym<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 598 def : BinRMWPatExternalSym<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 599 600 def : BinRMWPatExternalSym<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 601 def : BinRMWPatExternalSym<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 602 def : BinRMWPatExternalSym<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 603 def : BinRMWPatExternalSym<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 604 605 // Truncating-extending binary RMWs with just a constant offset 606 def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 607 def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 608 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 609 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 610 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 611 612 def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 613 def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 614 def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 615 def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 616 617 def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 618 def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 619 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 620 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 621 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 622 623 def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 624 def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 625 def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 626 def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 627 628 def : BinRMWPatExternSymOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 629 def : BinRMWPatExternSymOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 630 def : BinRMWPatExternSymOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 631 def : BinRMWPatExternSymOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 632 def : BinRMWPatExternSymOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 633 634 def : BinRMWPatExternSymOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 635 def : BinRMWPatExternSymOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 636 def : BinRMWPatExternSymOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 637 def : BinRMWPatExternSymOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 638} 639 640let Predicates = [HasAtomics] in { 641defm : BinRMWTruncExtPattern< 642 atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, 643 ATOMIC_RMW8_U_ADD_I32, ATOMIC_RMW16_U_ADD_I32, 644 ATOMIC_RMW8_U_ADD_I64, ATOMIC_RMW16_U_ADD_I64, ATOMIC_RMW32_U_ADD_I64>; 645defm : BinRMWTruncExtPattern< 646 atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, 647 ATOMIC_RMW8_U_SUB_I32, ATOMIC_RMW16_U_SUB_I32, 648 ATOMIC_RMW8_U_SUB_I64, ATOMIC_RMW16_U_SUB_I64, ATOMIC_RMW32_U_SUB_I64>; 649defm : BinRMWTruncExtPattern< 650 atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, 651 ATOMIC_RMW8_U_AND_I32, ATOMIC_RMW16_U_AND_I32, 652 ATOMIC_RMW8_U_AND_I64, ATOMIC_RMW16_U_AND_I64, ATOMIC_RMW32_U_AND_I64>; 653defm : BinRMWTruncExtPattern< 654 atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, 655 ATOMIC_RMW8_U_OR_I32, ATOMIC_RMW16_U_OR_I32, 656 ATOMIC_RMW8_U_OR_I64, ATOMIC_RMW16_U_OR_I64, ATOMIC_RMW32_U_OR_I64>; 657defm : BinRMWTruncExtPattern< 658 atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, 659 ATOMIC_RMW8_U_XOR_I32, ATOMIC_RMW16_U_XOR_I32, 660 ATOMIC_RMW8_U_XOR_I64, ATOMIC_RMW16_U_XOR_I64, ATOMIC_RMW32_U_XOR_I64>; 661defm : BinRMWTruncExtPattern< 662 atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, 663 ATOMIC_RMW8_U_XCHG_I32, ATOMIC_RMW16_U_XCHG_I32, 664 ATOMIC_RMW8_U_XCHG_I64, ATOMIC_RMW16_U_XCHG_I64, ATOMIC_RMW32_U_XCHG_I64>; 665} // Predicates = [HasAtomics] 666 667//===----------------------------------------------------------------------===// 668// Atomic ternary read-modify-writes 669//===----------------------------------------------------------------------===// 670 671// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 672// flag}. When we use the success flag or both values, we can't make use of i64 673// truncate/extend versions of instructions for now, which is suboptimal. 674// Consider adding a pass after instruction selection that optimizes this case 675// if it is frequent. 676 677let Defs = [ARGUMENTS] in { 678 679multiclass WebAssemblyTerRMW<WebAssemblyRegClass rc, string Name, int Opcode> { 680 defm "" : I<(outs rc:$dst), 681 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 682 rc:$new), 683 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 684 !strconcat(Name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new"), 685 !strconcat(Name, "\t${off}, ${p2align}"), Opcode>; 686} 687 688defm ATOMIC_RMW_CMPXCHG_I32 : 689 WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0xfe48>; 690defm ATOMIC_RMW_CMPXCHG_I64 : 691 WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0xfe49>; 692defm ATOMIC_RMW8_U_CMPXCHG_I32 : 693 WebAssemblyTerRMW<I32, "i32.atomic.rmw8_u.cmpxchg", 0xfe4a>; 694defm ATOMIC_RMW16_U_CMPXCHG_I32 : 695 WebAssemblyTerRMW<I32, "i32.atomic.rmw16_u.cmpxchg", 0xfe4b>; 696defm ATOMIC_RMW8_U_CMPXCHG_I64 : 697 WebAssemblyTerRMW<I64, "i64.atomic.rmw8_u.cmpxchg", 0xfe4c>; 698defm ATOMIC_RMW16_U_CMPXCHG_I64 : 699 WebAssemblyTerRMW<I64, "i64.atomic.rmw16_u.cmpxchg", 0xfe4d>; 700defm ATOMIC_RMW32_U_CMPXCHG_I64 : 701 WebAssemblyTerRMW<I64, "i64.atomic.rmw32_u.cmpxchg", 0xfe4e>; 702} 703 704// Select ternary RMWs with no constant offset. 705class TerRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 706 Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), 707 (inst 0, 0, I32:$addr, ty:$exp, ty:$new)>; 708 709// Select ternary RMWs with a constant offset. 710 711// Pattern with address + immediate offset 712class TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 713 Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), 714 (inst 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>; 715 716class TerRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 717 Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 718 ty:$exp, ty:$new)), 719 (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, ty:$new)>; 720 721class TerRMWPatExternalSym<ValueType ty, PatFrag kind, NI inst> : 722 Pat<(ty (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), 723 ty:$exp, ty:$new)), 724 (inst 0, texternalsym:$off, I32:$addr, ty:$exp, ty:$new)>; 725 726// Select ternary RMWs with just a constant offset. 727class TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 728 Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 729 (inst 0, imm:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 730 731class TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 732 Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 733 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 734 735class TerRMWPatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> : 736 Pat<(ty (kind (WebAssemblywrapper texternalsym:$off), ty:$exp, ty:$new)), 737 (inst 0, texternalsym:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 738 739// Patterns for various addressing modes. 740multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 741 NI inst_64> { 742 def : TerRMWPatNoOffset<i32, rmw_32, inst_32>; 743 def : TerRMWPatNoOffset<i64, rmw_64, inst_64>; 744 745 def : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 746 def : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 747 def : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 748 def : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 749 750 def : TerRMWPatGlobalAddr<i32, rmw_32, inst_32>; 751 def : TerRMWPatGlobalAddr<i64, rmw_64, inst_64>; 752 753 def : TerRMWPatExternalSym<i32, rmw_32, inst_32>; 754 def : TerRMWPatExternalSym<i64, rmw_64, inst_64>; 755 756 def : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; 757 def : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; 758 759 def : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 760 def : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 761 762 def : TerRMWPatExternSymOffOnly<i32, rmw_32, inst_32>; 763 def : TerRMWPatExternSymOffOnly<i64, rmw_64, inst_64>; 764} 765 766let Predicates = [HasAtomics] in { 767defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, 768 ATOMIC_RMW_CMPXCHG_I32, ATOMIC_RMW_CMPXCHG_I64>; 769} // Predicates = [HasAtomics] 770 771// Truncating & zero-extending ternary RMW patterns. 772// DAG legalization & optimization before instruction selection may introduce 773// additional nodes such as anyext or assertzext depending on operand types. 774class zext_ter_rmw_8_32<PatFrag kind> : 775 PatFrag<(ops node:$addr, node:$exp, node:$new), 776 (and (i32 (kind node:$addr, node:$exp, node:$new)), 255)>; 777class zext_ter_rmw_16_32<PatFrag kind> : 778 PatFrag<(ops node:$addr, node:$exp, node:$new), 779 (and (i32 (kind node:$addr, node:$exp, node:$new)), 65535)>; 780class zext_ter_rmw_8_64<PatFrag kind> : 781 PatFrag<(ops node:$addr, node:$exp, node:$new), 782 (zext (i32 (assertzext (i32 (kind node:$addr, 783 (i32 (trunc (i64 node:$exp))), 784 (i32 (trunc (i64 node:$new))))))))>; 785class zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 786class zext_ter_rmw_32_64<PatFrag kind> : 787 PatFrag<(ops node:$addr, node:$exp, node:$new), 788 (zext (i32 (kind node:$addr, 789 (i32 (trunc (i64 node:$exp))), 790 (i32 (trunc (i64 node:$new))))))>; 791 792// Truncating & sign-extending ternary RMW patterns. 793// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 794// zext RMW; the next instruction will be sext_inreg which is selected by 795// itself. 796class sext_ter_rmw_8_32<PatFrag kind> : 797 PatFrag<(ops node:$addr, node:$exp, node:$new), 798 (kind node:$addr, node:$exp, node:$new)>; 799class sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 800class sext_ter_rmw_8_64<PatFrag kind> : 801 PatFrag<(ops node:$addr, node:$exp, node:$new), 802 (anyext (i32 (assertzext (i32 803 (kind node:$addr, 804 (i32 (trunc (i64 node:$exp))), 805 (i32 (trunc (i64 node:$new))))))))>; 806class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 807// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_s/i32 808 809// Patterns for various addressing modes for truncating-extending ternary RMWs. 810multiclass TerRMWTruncExtPattern< 811 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 812 NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 813 // Truncating-extending ternary RMWs with no constant offset 814 def : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 815 def : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 816 def : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 817 def : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 818 def : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 819 820 def : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 821 def : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 822 def : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 823 def : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 824 825 // Truncating-extending ternary RMWs with a constant offset 826 def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 827 def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 828 def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 829 def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 830 def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 831 def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 832 def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 833 def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 834 def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 835 def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 836 837 def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 838 def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 839 def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 840 def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 841 def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 842 def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 843 def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 844 def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 845 846 def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 847 def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 848 def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 849 def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 850 def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 851 852 def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 853 def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 854 def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 855 def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 856 857 def : TerRMWPatExternalSym<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 858 def : TerRMWPatExternalSym<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 859 def : TerRMWPatExternalSym<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 860 def : TerRMWPatExternalSym<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 861 def : TerRMWPatExternalSym<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 862 863 def : TerRMWPatExternalSym<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 864 def : TerRMWPatExternalSym<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 865 def : TerRMWPatExternalSym<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 866 def : TerRMWPatExternalSym<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 867 868 // Truncating-extending ternary RMWs with just a constant offset 869 def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 870 def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 871 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 872 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 873 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 874 875 def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 876 def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 877 def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 878 def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 879 880 def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 881 def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 882 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 883 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 884 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 885 886 def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 887 def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 888 def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 889 def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 890 891 def : TerRMWPatExternSymOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 892 def : TerRMWPatExternSymOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 893 def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 894 def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 895 def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 896 897 def : TerRMWPatExternSymOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 898 def : TerRMWPatExternSymOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 899 def : TerRMWPatExternSymOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 900 def : TerRMWPatExternSymOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 901} 902 903let Predicates = [HasAtomics] in { 904defm : TerRMWTruncExtPattern< 905 atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, 906 ATOMIC_RMW8_U_CMPXCHG_I32, ATOMIC_RMW16_U_CMPXCHG_I32, 907 ATOMIC_RMW8_U_CMPXCHG_I64, ATOMIC_RMW16_U_CMPXCHG_I64, 908 ATOMIC_RMW32_U_CMPXCHG_I64>; 909} 910 911//===----------------------------------------------------------------------===// 912// Atomic wait / notify 913//===----------------------------------------------------------------------===// 914 915let Defs = [ARGUMENTS] in { 916let hasSideEffects = 1 in { 917defm ATOMIC_NOTIFY : 918 I<(outs I32:$dst), 919 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 920 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 921 "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 922 "atomic.notify \t${off}, ${p2align}", 0xfe00>; 923let mayLoad = 1 in { 924defm ATOMIC_WAIT_I32 : 925 I<(outs I32:$dst), 926 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, I64:$timeout), 927 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 928 "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 929 "i32.atomic.wait \t${off}, ${p2align}", 0xfe01>; 930defm ATOMIC_WAIT_I64 : 931 I<(outs I32:$dst), 932 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, I64:$timeout), 933 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 934 "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 935 "i64.atomic.wait \t${off}, ${p2align}", 0xfe02>; 936} // mayLoad = 1 937} // hasSideEffects = 1 938} // Defs = [ARGUMENTS] 939 940let Predicates = [HasAtomics] in { 941// Select notifys with no constant offset. 942class NotifyPatNoOffset<Intrinsic kind> : 943 Pat<(i32 (kind I32:$addr, I32:$count)), 944 (ATOMIC_NOTIFY 0, 0, I32:$addr, I32:$count)>; 945def : NotifyPatNoOffset<int_wasm_atomic_notify>; 946 947// Select notifys with a constant offset. 948 949// Pattern with address + immediate offset 950class NotifyPatImmOff<Intrinsic kind, PatFrag operand> : 951 Pat<(i32 (kind (operand I32:$addr, imm:$off), I32:$count)), 952 (ATOMIC_NOTIFY 0, imm:$off, I32:$addr, I32:$count)>; 953def : NotifyPatImmOff<int_wasm_atomic_notify, regPlusImm>; 954def : NotifyPatImmOff<int_wasm_atomic_notify, or_is_add>; 955 956class NotifyPatGlobalAddr<Intrinsic kind> : 957 Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 958 I32:$count)), 959 (ATOMIC_NOTIFY 0, tglobaladdr:$off, I32:$addr, I32:$count)>; 960def : NotifyPatGlobalAddr<int_wasm_atomic_notify>; 961 962class NotifyPatExternalSym<Intrinsic kind> : 963 Pat<(i32 (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), 964 I32:$count)), 965 (ATOMIC_NOTIFY 0, texternalsym:$off, I32:$addr, I32:$count)>; 966def : NotifyPatExternalSym<int_wasm_atomic_notify>; 967 968// Select notifys with just a constant offset. 969class NotifyPatOffsetOnly<Intrinsic kind> : 970 Pat<(i32 (kind imm:$off, I32:$count)), 971 (ATOMIC_NOTIFY 0, imm:$off, (CONST_I32 0), I32:$count)>; 972def : NotifyPatOffsetOnly<int_wasm_atomic_notify>; 973 974class NotifyPatGlobalAddrOffOnly<Intrinsic kind> : 975 Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), I32:$count)), 976 (ATOMIC_NOTIFY 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)>; 977def : NotifyPatGlobalAddrOffOnly<int_wasm_atomic_notify>; 978 979class NotifyPatExternSymOffOnly<Intrinsic kind> : 980 Pat<(i32 (kind (WebAssemblywrapper texternalsym:$off), I32:$count)), 981 (ATOMIC_NOTIFY 0, texternalsym:$off, (CONST_I32 0), I32:$count)>; 982def : NotifyPatExternSymOffOnly<int_wasm_atomic_notify>; 983 984// Select waits with no constant offset. 985class WaitPatNoOffset<ValueType ty, Intrinsic kind, NI inst> : 986 Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), 987 (inst 0, 0, I32:$addr, ty:$exp, I64:$timeout)>; 988def : WaitPatNoOffset<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 989def : WaitPatNoOffset<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 990 991// Select waits with a constant offset. 992 993// Pattern with address + immediate offset 994class WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, NI inst> : 995 Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), 996 (inst 0, imm:$off, I32:$addr, ty:$exp, I64:$timeout)>; 997def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, regPlusImm, ATOMIC_WAIT_I32>; 998def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, or_is_add, ATOMIC_WAIT_I32>; 999def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, regPlusImm, ATOMIC_WAIT_I64>; 1000def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, or_is_add, ATOMIC_WAIT_I64>; 1001 1002class WaitPatGlobalAddr<ValueType ty, Intrinsic kind, NI inst> : 1003 Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 1004 ty:$exp, I64:$timeout)), 1005 (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, I64:$timeout)>; 1006def : WaitPatGlobalAddr<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1007def : WaitPatGlobalAddr<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1008 1009class WaitPatExternalSym<ValueType ty, Intrinsic kind, NI inst> : 1010 Pat<(i32 (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), 1011 ty:$exp, I64:$timeout)), 1012 (inst 0, texternalsym:$off, I32:$addr, ty:$exp, I64:$timeout)>; 1013def : WaitPatExternalSym<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1014def : WaitPatExternalSym<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1015 1016// Select wait_i32, ATOMIC_WAIT_I32s with just a constant offset. 1017class WaitPatOffsetOnly<ValueType ty, Intrinsic kind, NI inst> : 1018 Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 1019 (inst 0, imm:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 1020def : WaitPatOffsetOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1021def : WaitPatOffsetOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1022 1023class WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, NI inst> : 1024 Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, I64:$timeout)), 1025 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 1026def : WaitPatGlobalAddrOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1027def : WaitPatGlobalAddrOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1028 1029class WaitPatExternSymOffOnly<ValueType ty, Intrinsic kind, NI inst> : 1030 Pat<(i32 (kind (WebAssemblywrapper texternalsym:$off), ty:$exp, 1031 I64:$timeout)), 1032 (inst 0, texternalsym:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 1033def : WaitPatExternSymOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1034def : WaitPatExternSymOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1035} // Predicates = [HasAtomics] 1036