1// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9/// 10/// \file 11/// WebAssembly Atomic operand code-gen constructs. 12/// 13//===----------------------------------------------------------------------===// 14 15//===----------------------------------------------------------------------===// 16// Atomic loads 17//===----------------------------------------------------------------------===// 18 19let Defs = [ARGUMENTS] in { 20defm ATOMIC_LOAD_I32 : WebAssemblyLoad<I32, "i32.atomic.load", 0xfe10>; 21defm ATOMIC_LOAD_I64 : WebAssemblyLoad<I64, "i64.atomic.load", 0xfe11>; 22} // Defs = [ARGUMENTS] 23 24// Select loads with no constant offset. 25let Predicates = [HasAtomics] in { 26def : LoadPatNoOffset<i32, atomic_load_32, ATOMIC_LOAD_I32>; 27def : LoadPatNoOffset<i64, atomic_load_64, ATOMIC_LOAD_I64>; 28 29// Select loads with a constant offset. 30 31// Pattern with address + immediate offset 32def : LoadPatImmOff<i32, atomic_load_32, regPlusImm, ATOMIC_LOAD_I32>; 33def : LoadPatImmOff<i64, atomic_load_64, regPlusImm, ATOMIC_LOAD_I64>; 34def : LoadPatImmOff<i32, atomic_load_32, or_is_add, ATOMIC_LOAD_I32>; 35def : LoadPatImmOff<i64, atomic_load_64, or_is_add, ATOMIC_LOAD_I64>; 36 37def : LoadPatGlobalAddr<i32, atomic_load_32, ATOMIC_LOAD_I32>; 38def : LoadPatGlobalAddr<i64, atomic_load_64, ATOMIC_LOAD_I64>; 39 40def : LoadPatExternalSym<i32, atomic_load_32, ATOMIC_LOAD_I32>; 41def : LoadPatExternalSym<i64, atomic_load_64, ATOMIC_LOAD_I64>; 42 43// Select loads with just a constant offset. 44def : LoadPatOffsetOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 45def : LoadPatOffsetOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 46 47def : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 48def : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 49 50def : LoadPatExternSymOffOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 51def : LoadPatExternSymOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 52 53} // Predicates = [HasAtomics] 54 55// Extending loads. Note that there are only zero-extending atomic loads, no 56// sign-extending loads. 57let Defs = [ARGUMENTS] in { 58defm ATOMIC_LOAD8_U_I32 : WebAssemblyLoad<I32, "i32.atomic.load8_u", 0xfe12>; 59defm ATOMIC_LOAD16_U_I32 : WebAssemblyLoad<I32, "i32.atomic.load16_u", 0xfe13>; 60defm ATOMIC_LOAD8_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load8_u", 0xfe14>; 61defm ATOMIC_LOAD16_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load16_u", 0xfe15>; 62defm ATOMIC_LOAD32_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load32_u", 0xfe16>; 63} // Defs = [ARGUMENTS] 64 65// Fragments for extending loads. These are different from regular loads because 66// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 67// therefore don't have the extension type field. So instead of matching that, 68// we match the patterns that the type legalizer expands them to. 69 70// We directly match zext patterns and select the zext atomic loads. 71// i32 (zext (i8 (atomic_load_8))) gets legalized to 72// i32 (and (i32 (atomic_load_8)), 255) 73// These can be selected to a single zero-extending atomic load instruction. 74def zext_aload_8_32 : 75 PatFrag<(ops node:$addr), (and (i32 (atomic_load_8 node:$addr)), 255)>; 76def zext_aload_16_32 : 77 PatFrag<(ops node:$addr), (and (i32 (atomic_load_16 node:$addr)), 65535)>; 78// Unlike regular loads, extension to i64 is handled differently than i32. 79// i64 (zext (i8 (atomic_load_8))) gets legalized to 80// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 81def zext_aload_8_64 : 82 PatFrag<(ops node:$addr), 83 (and (i64 (anyext (i32 (atomic_load_8 node:$addr)))), 255)>; 84def zext_aload_16_64 : 85 PatFrag<(ops node:$addr), 86 (and (i64 (anyext (i32 (atomic_load_16 node:$addr)))), 65535)>; 87def zext_aload_32_64 : 88 PatFrag<(ops node:$addr), 89 (zext (i32 (atomic_load node:$addr)))>; 90 91// We don't have single sext atomic load instructions. So for sext loads, we 92// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 93// results) and select a zext load; the next instruction will be sext_inreg 94// which is selected by itself. 95def sext_aload_8_64 : 96 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 97def sext_aload_16_64 : 98 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 99 100let Predicates = [HasAtomics] in { 101// Select zero-extending loads with no constant offset. 102def : LoadPatNoOffset<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 103def : LoadPatNoOffset<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 104def : LoadPatNoOffset<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 105def : LoadPatNoOffset<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 106def : LoadPatNoOffset<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 107 108// Select sign-extending loads with no constant offset 109def : LoadPatNoOffset<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 110def : LoadPatNoOffset<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 111def : LoadPatNoOffset<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 112def : LoadPatNoOffset<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 113// 32->64 sext load gets selected as i32.atomic.load, i64.extend_s/i32 114 115// Zero-extending loads with constant offset 116def : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, ATOMIC_LOAD8_U_I32>; 117def : LoadPatImmOff<i32, zext_aload_16_32, regPlusImm, ATOMIC_LOAD16_U_I32>; 118def : LoadPatImmOff<i32, zext_aload_8_32, or_is_add, ATOMIC_LOAD8_U_I32>; 119def : LoadPatImmOff<i32, zext_aload_16_32, or_is_add, ATOMIC_LOAD16_U_I32>; 120def : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 121def : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 122def : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, ATOMIC_LOAD32_U_I64>; 123def : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 124def : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 125def : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, ATOMIC_LOAD32_U_I64>; 126 127// Sign-extending loads with constant offset 128def : LoadPatImmOff<i32, atomic_load_8, regPlusImm, ATOMIC_LOAD8_U_I32>; 129def : LoadPatImmOff<i32, atomic_load_16, regPlusImm, ATOMIC_LOAD16_U_I32>; 130def : LoadPatImmOff<i32, atomic_load_8, or_is_add, ATOMIC_LOAD8_U_I32>; 131def : LoadPatImmOff<i32, atomic_load_16, or_is_add, ATOMIC_LOAD16_U_I32>; 132def : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 133def : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 134def : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 135def : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 136// No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 137 138def : LoadPatGlobalAddr<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 139def : LoadPatGlobalAddr<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 140def : LoadPatGlobalAddr<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 141def : LoadPatGlobalAddr<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 142def : LoadPatGlobalAddr<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 143def : LoadPatGlobalAddr<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 144def : LoadPatGlobalAddr<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 145def : LoadPatGlobalAddr<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 146def : LoadPatGlobalAddr<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 147 148def : LoadPatExternalSym<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 149def : LoadPatExternalSym<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 150def : LoadPatExternalSym<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 151def : LoadPatExternalSym<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 152def : LoadPatExternalSym<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 153def : LoadPatExternalSym<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 154def : LoadPatExternalSym<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 155def : LoadPatExternalSym<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 156def : LoadPatExternalSym<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 157 158// Extending loads with just a constant offset 159def : LoadPatOffsetOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 160def : LoadPatOffsetOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 161def : LoadPatOffsetOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 162def : LoadPatOffsetOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 163def : LoadPatOffsetOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 164def : LoadPatOffsetOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 165def : LoadPatOffsetOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 166def : LoadPatOffsetOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 167def : LoadPatOffsetOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 168 169def : LoadPatGlobalAddrOffOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 170def : LoadPatGlobalAddrOffOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 171def : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 172def : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 173def : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 174def : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 175def : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 176def : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 177def : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 178 179def : LoadPatExternSymOffOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 180def : LoadPatExternSymOffOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 181def : LoadPatExternSymOffOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 182def : LoadPatExternSymOffOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 183def : LoadPatExternSymOffOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 184def : LoadPatExternSymOffOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 185def : LoadPatExternSymOffOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 186def : LoadPatExternSymOffOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 187def : LoadPatExternSymOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 188 189} // Predicates = [HasAtomics] 190 191//===----------------------------------------------------------------------===// 192// Atomic stores 193//===----------------------------------------------------------------------===// 194 195let Defs = [ARGUMENTS] in { 196defm ATOMIC_STORE_I32 : WebAssemblyStore<I32, "i32.atomic.store", 0xfe17>; 197defm ATOMIC_STORE_I64 : WebAssemblyStore<I64, "i64.atomic.store", 0xfe18>; 198} // Defs = [ARGUMENTS] 199 200// We need an 'atomic' version of store patterns because store and atomic_store 201// nodes have different operand orders: 202// store: (store $val, $ptr) 203// atomic_store: (store $ptr, $val) 204 205let Predicates = [HasAtomics] in { 206 207// Select stores with no constant offset. 208class AStorePatNoOffset<ValueType ty, PatFrag kind, NI inst> : 209 Pat<(kind I32:$addr, ty:$val), (inst 0, 0, I32:$addr, ty:$val)>; 210def : AStorePatNoOffset<i32, atomic_store_32, ATOMIC_STORE_I32>; 211def : AStorePatNoOffset<i64, atomic_store_64, ATOMIC_STORE_I64>; 212 213// Select stores with a constant offset. 214 215// Pattern with address + immediate offset 216class AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 217 Pat<(kind (operand I32:$addr, imm:$off), ty:$val), 218 (inst 0, imm:$off, I32:$addr, ty:$val)>; 219def : AStorePatImmOff<i32, atomic_store_32, regPlusImm, ATOMIC_STORE_I32>; 220def : AStorePatImmOff<i64, atomic_store_64, regPlusImm, ATOMIC_STORE_I64>; 221def : AStorePatImmOff<i32, atomic_store_32, or_is_add, ATOMIC_STORE_I32>; 222def : AStorePatImmOff<i64, atomic_store_64, or_is_add, ATOMIC_STORE_I64>; 223 224class AStorePatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 225 Pat<(kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 226 ty:$val), 227 (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>; 228def : AStorePatGlobalAddr<i32, atomic_store_32, ATOMIC_STORE_I32>; 229def : AStorePatGlobalAddr<i64, atomic_store_64, ATOMIC_STORE_I64>; 230 231class AStorePatExternalSym<ValueType ty, PatFrag kind, NI inst> : 232 Pat<(kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), ty:$val), 233 (inst 0, texternalsym:$off, I32:$addr, ty:$val)>; 234def : AStorePatExternalSym<i32, atomic_store_32, ATOMIC_STORE_I32>; 235def : AStorePatExternalSym<i64, atomic_store_64, ATOMIC_STORE_I64>; 236 237// Select stores with just a constant offset. 238class AStorePatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 239 Pat<(kind imm:$off, ty:$val), (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 240def : AStorePatOffsetOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 241def : AStorePatOffsetOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 242 243class AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 244 Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 245 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 246def : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 247def : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 248 249class AStorePatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> : 250 Pat<(kind (WebAssemblywrapper texternalsym:$off), ty:$val), 251 (inst 0, texternalsym:$off, (CONST_I32 0), ty:$val)>; 252def : AStorePatExternSymOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 253def : AStorePatExternSymOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 254 255} // Predicates = [HasAtomics] 256 257// Truncating stores. 258let Defs = [ARGUMENTS] in { 259defm ATOMIC_STORE8_I32 : WebAssemblyStore<I32, "i32.atomic.store8", 0xfe19>; 260defm ATOMIC_STORE16_I32 : WebAssemblyStore<I32, "i32.atomic.store16", 0xfe1a>; 261defm ATOMIC_STORE8_I64 : WebAssemblyStore<I64, "i64.atomic.store8", 0xfe1b>; 262defm ATOMIC_STORE16_I64 : WebAssemblyStore<I64, "i64.atomic.store16", 0xfe1c>; 263defm ATOMIC_STORE32_I64 : WebAssemblyStore<I64, "i64.atomic.store32", 0xfe1d>; 264} // Defs = [ARGUMENTS] 265 266// Fragments for truncating stores. 267 268// We don't have single truncating atomic store instructions. For 32-bit 269// instructions, we just need to match bare atomic stores. On the other hand, 270// truncating stores from i64 values are once truncated to i32 first. 271class trunc_astore_64<PatFrag kind> : 272 PatFrag<(ops node:$addr, node:$val), 273 (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 274def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 275def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 276def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 277 278let Predicates = [HasAtomics] in { 279 280// Truncating stores with no constant offset 281def : AStorePatNoOffset<i32, atomic_store_8, ATOMIC_STORE8_I32>; 282def : AStorePatNoOffset<i32, atomic_store_16, ATOMIC_STORE16_I32>; 283def : AStorePatNoOffset<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 284def : AStorePatNoOffset<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 285def : AStorePatNoOffset<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 286 287// Truncating stores with a constant offset 288def : AStorePatImmOff<i32, atomic_store_8, regPlusImm, ATOMIC_STORE8_I32>; 289def : AStorePatImmOff<i32, atomic_store_16, regPlusImm, ATOMIC_STORE16_I32>; 290def : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, ATOMIC_STORE8_I64>; 291def : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, ATOMIC_STORE16_I64>; 292def : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, ATOMIC_STORE32_I64>; 293def : AStorePatImmOff<i32, atomic_store_8, or_is_add, ATOMIC_STORE8_I32>; 294def : AStorePatImmOff<i32, atomic_store_16, or_is_add, ATOMIC_STORE16_I32>; 295def : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, ATOMIC_STORE8_I64>; 296def : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, ATOMIC_STORE16_I64>; 297def : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, ATOMIC_STORE32_I64>; 298 299def : AStorePatGlobalAddr<i32, atomic_store_8, ATOMIC_STORE8_I32>; 300def : AStorePatGlobalAddr<i32, atomic_store_16, ATOMIC_STORE16_I32>; 301def : AStorePatGlobalAddr<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 302def : AStorePatGlobalAddr<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 303def : AStorePatGlobalAddr<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 304 305def : AStorePatExternalSym<i32, atomic_store_8, ATOMIC_STORE8_I32>; 306def : AStorePatExternalSym<i32, atomic_store_16, ATOMIC_STORE16_I32>; 307def : AStorePatExternalSym<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 308def : AStorePatExternalSym<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 309def : AStorePatExternalSym<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 310 311// Truncating stores with just a constant offset 312def : AStorePatOffsetOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 313def : AStorePatOffsetOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 314def : AStorePatOffsetOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 315def : AStorePatOffsetOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 316def : AStorePatOffsetOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 317 318def : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 319def : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 320def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 321def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 322def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 323 324def : AStorePatExternSymOffOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 325def : AStorePatExternSymOffOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 326def : AStorePatExternSymOffOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 327def : AStorePatExternSymOffOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 328def : AStorePatExternSymOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 329 330} // Predicates = [HasAtomics] 331 332//===----------------------------------------------------------------------===// 333// Atomic binary read-modify-writes 334//===----------------------------------------------------------------------===// 335 336let Defs = [ARGUMENTS] in { 337 338multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string Name, int Opcode> { 339 defm "" : I<(outs rc:$dst), 340 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 341 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 342 !strconcat(Name, "\t$dst, ${off}(${addr})${p2align}, $val"), 343 !strconcat(Name, "\t${off}, ${p2align}"), Opcode>; 344} 345 346defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0xfe1e>; 347defm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0xfe1f>; 348defm ATOMIC_RMW8_U_ADD_I32 : 349 WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.add", 0xfe20>; 350defm ATOMIC_RMW16_U_ADD_I32 : 351 WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.add", 0xfe21>; 352defm ATOMIC_RMW8_U_ADD_I64 : 353 WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.add", 0xfe22>; 354defm ATOMIC_RMW16_U_ADD_I64 : 355 WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.add", 0xfe23>; 356defm ATOMIC_RMW32_U_ADD_I64 : 357 WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.add", 0xfe24>; 358 359defm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0xfe25>; 360defm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0xfe26>; 361defm ATOMIC_RMW8_U_SUB_I32 : 362 WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.sub", 0xfe27>; 363defm ATOMIC_RMW16_U_SUB_I32 : 364 WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.sub", 0xfe28>; 365defm ATOMIC_RMW8_U_SUB_I64 : 366 WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.sub", 0xfe29>; 367defm ATOMIC_RMW16_U_SUB_I64 : 368 WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.sub", 0xfe2a>; 369defm ATOMIC_RMW32_U_SUB_I64 : 370 WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.sub", 0xfe2b>; 371 372defm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0xfe2c>; 373defm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0xfe2d>; 374defm ATOMIC_RMW8_U_AND_I32 : 375 WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.and", 0xfe2e>; 376defm ATOMIC_RMW16_U_AND_I32 : 377 WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.and", 0xfe2f>; 378defm ATOMIC_RMW8_U_AND_I64 : 379 WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.and", 0xfe30>; 380defm ATOMIC_RMW16_U_AND_I64 : 381 WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.and", 0xfe31>; 382defm ATOMIC_RMW32_U_AND_I64 : 383 WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.and", 0xfe32>; 384 385defm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0xfe33>; 386defm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0xfe34>; 387defm ATOMIC_RMW8_U_OR_I32 : 388 WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.or", 0xfe35>; 389defm ATOMIC_RMW16_U_OR_I32 : 390 WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.or", 0xfe36>; 391defm ATOMIC_RMW8_U_OR_I64 : 392 WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.or", 0xfe37>; 393defm ATOMIC_RMW16_U_OR_I64 : 394 WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.or", 0xfe38>; 395defm ATOMIC_RMW32_U_OR_I64 : 396 WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.or", 0xfe39>; 397 398defm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0xfe3a>; 399defm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0xfe3b>; 400defm ATOMIC_RMW8_U_XOR_I32 : 401 WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.xor", 0xfe3c>; 402defm ATOMIC_RMW16_U_XOR_I32 : 403 WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.xor", 0xfe3d>; 404defm ATOMIC_RMW8_U_XOR_I64 : 405 WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.xor", 0xfe3e>; 406defm ATOMIC_RMW16_U_XOR_I64 : 407 WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.xor", 0xfe3f>; 408defm ATOMIC_RMW32_U_XOR_I64 : 409 WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.xor", 0xfe40>; 410 411defm ATOMIC_RMW_XCHG_I32 : 412 WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0xfe41>; 413defm ATOMIC_RMW_XCHG_I64 : 414 WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0xfe42>; 415defm ATOMIC_RMW8_U_XCHG_I32 : 416 WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.xchg", 0xfe43>; 417defm ATOMIC_RMW16_U_XCHG_I32 : 418 WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.xchg", 0xfe44>; 419defm ATOMIC_RMW8_U_XCHG_I64 : 420 WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.xchg", 0xfe45>; 421defm ATOMIC_RMW16_U_XCHG_I64 : 422 WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.xchg", 0xfe46>; 423defm ATOMIC_RMW32_U_XCHG_I64 : 424 WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.xchg", 0xfe47>; 425} 426 427// Select binary RMWs with no constant offset. 428class BinRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 429 Pat<(ty (kind I32:$addr, ty:$val)), (inst 0, 0, I32:$addr, ty:$val)>; 430 431// Select binary RMWs with a constant offset. 432 433// Pattern with address + immediate offset 434class BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 435 Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), 436 (inst 0, imm:$off, I32:$addr, ty:$val)>; 437 438class BinRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 439 Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 440 ty:$val)), 441 (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>; 442 443class BinRMWPatExternalSym<ValueType ty, PatFrag kind, NI inst> : 444 Pat<(ty (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), 445 ty:$val)), 446 (inst 0, texternalsym:$off, I32:$addr, ty:$val)>; 447 448// Select binary RMWs with just a constant offset. 449class BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 450 Pat<(ty (kind imm:$off, ty:$val)), 451 (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 452 453class BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 454 Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 455 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 456 457class BinRMWPatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> : 458 Pat<(ty (kind (WebAssemblywrapper texternalsym:$off), ty:$val)), 459 (inst 0, texternalsym:$off, (CONST_I32 0), ty:$val)>; 460 461// Patterns for various addressing modes. 462multiclass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 463 NI inst_64> { 464 def : BinRMWPatNoOffset<i32, rmw_32, inst_32>; 465 def : BinRMWPatNoOffset<i64, rmw_64, inst_64>; 466 467 def : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 468 def : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 469 def : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 470 def : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 471 472 def : BinRMWPatGlobalAddr<i32, rmw_32, inst_32>; 473 def : BinRMWPatGlobalAddr<i64, rmw_64, inst_64>; 474 475 def : BinRMWPatExternalSym<i32, rmw_32, inst_32>; 476 def : BinRMWPatExternalSym<i64, rmw_64, inst_64>; 477 478 def : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>; 479 def : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>; 480 481 def : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 482 def : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 483 484 def : BinRMWPatExternSymOffOnly<i32, rmw_32, inst_32>; 485 def : BinRMWPatExternSymOffOnly<i64, rmw_64, inst_64>; 486} 487 488let Predicates = [HasAtomics] in { 489defm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, ATOMIC_RMW_ADD_I32, 490 ATOMIC_RMW_ADD_I64>; 491defm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, ATOMIC_RMW_SUB_I32, 492 ATOMIC_RMW_SUB_I64>; 493defm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, ATOMIC_RMW_AND_I32, 494 ATOMIC_RMW_AND_I64>; 495defm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, ATOMIC_RMW_OR_I32, 496 ATOMIC_RMW_OR_I64>; 497defm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, ATOMIC_RMW_XOR_I32, 498 ATOMIC_RMW_XOR_I64>; 499defm : BinRMWPattern<atomic_swap_32, atomic_swap_64, ATOMIC_RMW_XCHG_I32, 500 ATOMIC_RMW_XCHG_I64>; 501} // Predicates = [HasAtomics] 502 503// Truncating & zero-extending binary RMW patterns. 504// These are combined patterns of truncating store patterns and zero-extending 505// load patterns above. 506class zext_bin_rmw_8_32<PatFrag kind> : 507 PatFrag<(ops node:$addr, node:$val), 508 (and (i32 (kind node:$addr, node:$val)), 255)>; 509class zext_bin_rmw_16_32<PatFrag kind> : 510 PatFrag<(ops node:$addr, node:$val), 511 (and (i32 (kind node:$addr, node:$val)), 65535)>; 512class zext_bin_rmw_8_64<PatFrag kind> : 513 PatFrag<(ops node:$addr, node:$val), 514 (and (i64 (anyext (i32 (kind node:$addr, 515 (i32 (trunc (i64 node:$val))))))), 255)>; 516class zext_bin_rmw_16_64<PatFrag kind> : 517 PatFrag<(ops node:$addr, node:$val), 518 (and (i64 (anyext (i32 (kind node:$addr, 519 (i32 (trunc (i64 node:$val))))))), 65535)>; 520class zext_bin_rmw_32_64<PatFrag kind> : 521 PatFrag<(ops node:$addr, node:$val), 522 (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 523 524// Truncating & sign-extending binary RMW patterns. 525// These are combined patterns of truncating store patterns and sign-extending 526// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 527// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 528// is selected by itself. 529class sext_bin_rmw_8_32<PatFrag kind> : 530 PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 531class sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 532class sext_bin_rmw_8_64<PatFrag kind> : 533 PatFrag<(ops node:$addr, node:$val), 534 (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 535class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 536// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_s/i32 537 538// Patterns for various addressing modes for truncating-extending binary RMWs. 539multiclass BinRMWTruncExtPattern< 540 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 541 NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 542 // Truncating-extending binary RMWs with no constant offset 543 def : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 544 def : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 545 def : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 546 def : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 547 def : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 548 549 def : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 550 def : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 551 def : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 552 def : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 553 554 // Truncating-extending binary RMWs with a constant offset 555 def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 556 def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 557 def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 558 def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 559 def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 560 def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 561 def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 562 def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 563 def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 564 def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 565 566 def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 567 def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 568 def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 569 def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 570 def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 571 def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 572 def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 573 def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 574 575 def : BinRMWPatGlobalAddr<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 576 def : BinRMWPatGlobalAddr<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 577 def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 578 def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 579 def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 580 581 def : BinRMWPatGlobalAddr<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 582 def : BinRMWPatGlobalAddr<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 583 def : BinRMWPatGlobalAddr<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 584 def : BinRMWPatGlobalAddr<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 585 586 def : BinRMWPatExternalSym<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 587 def : BinRMWPatExternalSym<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 588 def : BinRMWPatExternalSym<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 589 def : BinRMWPatExternalSym<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 590 def : BinRMWPatExternalSym<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 591 592 def : BinRMWPatExternalSym<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 593 def : BinRMWPatExternalSym<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 594 def : BinRMWPatExternalSym<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 595 def : BinRMWPatExternalSym<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 596 597 // Truncating-extending binary RMWs with just a constant offset 598 def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 599 def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 600 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 601 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 602 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 603 604 def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 605 def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 606 def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 607 def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 608 609 def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 610 def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 611 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 612 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 613 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 614 615 def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 616 def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 617 def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 618 def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 619 620 def : BinRMWPatExternSymOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 621 def : BinRMWPatExternSymOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 622 def : BinRMWPatExternSymOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 623 def : BinRMWPatExternSymOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 624 def : BinRMWPatExternSymOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 625 626 def : BinRMWPatExternSymOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 627 def : BinRMWPatExternSymOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 628 def : BinRMWPatExternSymOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 629 def : BinRMWPatExternSymOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 630} 631 632let Predicates = [HasAtomics] in { 633defm : BinRMWTruncExtPattern< 634 atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, 635 ATOMIC_RMW8_U_ADD_I32, ATOMIC_RMW16_U_ADD_I32, 636 ATOMIC_RMW8_U_ADD_I64, ATOMIC_RMW16_U_ADD_I64, ATOMIC_RMW32_U_ADD_I64>; 637defm : BinRMWTruncExtPattern< 638 atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, 639 ATOMIC_RMW8_U_SUB_I32, ATOMIC_RMW16_U_SUB_I32, 640 ATOMIC_RMW8_U_SUB_I64, ATOMIC_RMW16_U_SUB_I64, ATOMIC_RMW32_U_SUB_I64>; 641defm : BinRMWTruncExtPattern< 642 atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, 643 ATOMIC_RMW8_U_AND_I32, ATOMIC_RMW16_U_AND_I32, 644 ATOMIC_RMW8_U_AND_I64, ATOMIC_RMW16_U_AND_I64, ATOMIC_RMW32_U_AND_I64>; 645defm : BinRMWTruncExtPattern< 646 atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, 647 ATOMIC_RMW8_U_OR_I32, ATOMIC_RMW16_U_OR_I32, 648 ATOMIC_RMW8_U_OR_I64, ATOMIC_RMW16_U_OR_I64, ATOMIC_RMW32_U_OR_I64>; 649defm : BinRMWTruncExtPattern< 650 atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, 651 ATOMIC_RMW8_U_XOR_I32, ATOMIC_RMW16_U_XOR_I32, 652 ATOMIC_RMW8_U_XOR_I64, ATOMIC_RMW16_U_XOR_I64, ATOMIC_RMW32_U_XOR_I64>; 653defm : BinRMWTruncExtPattern< 654 atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, 655 ATOMIC_RMW8_U_XCHG_I32, ATOMIC_RMW16_U_XCHG_I32, 656 ATOMIC_RMW8_U_XCHG_I64, ATOMIC_RMW16_U_XCHG_I64, ATOMIC_RMW32_U_XCHG_I64>; 657} // Predicates = [HasAtomics] 658 659//===----------------------------------------------------------------------===// 660// Atomic ternary read-modify-writes 661//===----------------------------------------------------------------------===// 662 663// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 664// flag}. When we use the success flag or both values, we can't make use of i64 665// truncate/extend versions of instructions for now, which is suboptimal. 666// Consider adding a pass after instruction selection that optimizes this case 667// if it is frequent. 668 669let Defs = [ARGUMENTS] in { 670 671multiclass WebAssemblyTerRMW<WebAssemblyRegClass rc, string Name, int Opcode> { 672 defm "" : I<(outs rc:$dst), 673 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 674 rc:$new), 675 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 676 !strconcat(Name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new"), 677 !strconcat(Name, "\t${off}, ${p2align}"), Opcode>; 678} 679 680defm ATOMIC_RMW_CMPXCHG_I32 : 681 WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0xfe48>; 682defm ATOMIC_RMW_CMPXCHG_I64 : 683 WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0xfe49>; 684defm ATOMIC_RMW8_U_CMPXCHG_I32 : 685 WebAssemblyTerRMW<I32, "i32.atomic.rmw8_u.cmpxchg", 0xfe4a>; 686defm ATOMIC_RMW16_U_CMPXCHG_I32 : 687 WebAssemblyTerRMW<I32, "i32.atomic.rmw16_u.cmpxchg", 0xfe4b>; 688defm ATOMIC_RMW8_U_CMPXCHG_I64 : 689 WebAssemblyTerRMW<I64, "i64.atomic.rmw8_u.cmpxchg", 0xfe4c>; 690defm ATOMIC_RMW16_U_CMPXCHG_I64 : 691 WebAssemblyTerRMW<I64, "i64.atomic.rmw16_u.cmpxchg", 0xfe4d>; 692defm ATOMIC_RMW32_U_CMPXCHG_I64 : 693 WebAssemblyTerRMW<I64, "i64.atomic.rmw32_u.cmpxchg", 0xfe4e>; 694} 695 696// Select ternary RMWs with no constant offset. 697class TerRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 698 Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), 699 (inst 0, 0, I32:$addr, ty:$exp, ty:$new)>; 700 701// Select ternary RMWs with a constant offset. 702 703// Pattern with address + immediate offset 704class TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 705 Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), 706 (inst 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>; 707 708class TerRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 709 Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 710 ty:$exp, ty:$new)), 711 (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, ty:$new)>; 712 713class TerRMWPatExternalSym<ValueType ty, PatFrag kind, NI inst> : 714 Pat<(ty (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), 715 ty:$exp, ty:$new)), 716 (inst 0, texternalsym:$off, I32:$addr, ty:$exp, ty:$new)>; 717 718// Select ternary RMWs with just a constant offset. 719class TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 720 Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 721 (inst 0, imm:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 722 723class TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 724 Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 725 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 726 727class TerRMWPatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> : 728 Pat<(ty (kind (WebAssemblywrapper texternalsym:$off), ty:$exp, ty:$new)), 729 (inst 0, texternalsym:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 730 731// Patterns for various addressing modes. 732multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 733 NI inst_64> { 734 def : TerRMWPatNoOffset<i32, rmw_32, inst_32>; 735 def : TerRMWPatNoOffset<i64, rmw_64, inst_64>; 736 737 def : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 738 def : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 739 def : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 740 def : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 741 742 def : TerRMWPatGlobalAddr<i32, rmw_32, inst_32>; 743 def : TerRMWPatGlobalAddr<i64, rmw_64, inst_64>; 744 745 def : TerRMWPatExternalSym<i32, rmw_32, inst_32>; 746 def : TerRMWPatExternalSym<i64, rmw_64, inst_64>; 747 748 def : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; 749 def : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; 750 751 def : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 752 def : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 753 754 def : TerRMWPatExternSymOffOnly<i32, rmw_32, inst_32>; 755 def : TerRMWPatExternSymOffOnly<i64, rmw_64, inst_64>; 756} 757 758let Predicates = [HasAtomics] in { 759defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, 760 ATOMIC_RMW_CMPXCHG_I32, ATOMIC_RMW_CMPXCHG_I64>; 761} // Predicates = [HasAtomics] 762 763// Truncating & zero-extending ternary RMW patterns. 764// DAG legalization & optimization before instruction selection may introduce 765// additional nodes such as anyext or assertzext depending on operand types. 766class zext_ter_rmw_8_32<PatFrag kind> : 767 PatFrag<(ops node:$addr, node:$exp, node:$new), 768 (and (i32 (kind node:$addr, node:$exp, node:$new)), 255)>; 769class zext_ter_rmw_16_32<PatFrag kind> : 770 PatFrag<(ops node:$addr, node:$exp, node:$new), 771 (and (i32 (kind node:$addr, node:$exp, node:$new)), 65535)>; 772class zext_ter_rmw_8_64<PatFrag kind> : 773 PatFrag<(ops node:$addr, node:$exp, node:$new), 774 (zext (i32 (assertzext (i32 (kind node:$addr, 775 (i32 (trunc (i64 node:$exp))), 776 (i32 (trunc (i64 node:$new))))))))>; 777class zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 778class zext_ter_rmw_32_64<PatFrag kind> : 779 PatFrag<(ops node:$addr, node:$exp, node:$new), 780 (zext (i32 (kind node:$addr, 781 (i32 (trunc (i64 node:$exp))), 782 (i32 (trunc (i64 node:$new))))))>; 783 784// Truncating & sign-extending ternary RMW patterns. 785// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 786// zext RMW; the next instruction will be sext_inreg which is selected by 787// itself. 788class sext_ter_rmw_8_32<PatFrag kind> : 789 PatFrag<(ops node:$addr, node:$exp, node:$new), 790 (kind node:$addr, node:$exp, node:$new)>; 791class sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 792class sext_ter_rmw_8_64<PatFrag kind> : 793 PatFrag<(ops node:$addr, node:$exp, node:$new), 794 (anyext (i32 (assertzext (i32 795 (kind node:$addr, 796 (i32 (trunc (i64 node:$exp))), 797 (i32 (trunc (i64 node:$new))))))))>; 798class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 799// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_s/i32 800 801// Patterns for various addressing modes for truncating-extending ternary RMWs. 802multiclass TerRMWTruncExtPattern< 803 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 804 NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 805 // Truncating-extending ternary RMWs with no constant offset 806 def : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 807 def : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 808 def : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 809 def : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 810 def : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 811 812 def : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 813 def : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 814 def : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 815 def : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 816 817 // Truncating-extending ternary RMWs with a constant offset 818 def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 819 def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 820 def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 821 def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 822 def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 823 def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 824 def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 825 def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 826 def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 827 def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 828 829 def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 830 def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 831 def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 832 def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 833 def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 834 def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 835 def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 836 def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 837 838 def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 839 def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 840 def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 841 def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 842 def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 843 844 def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 845 def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 846 def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 847 def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 848 849 def : TerRMWPatExternalSym<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 850 def : TerRMWPatExternalSym<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 851 def : TerRMWPatExternalSym<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 852 def : TerRMWPatExternalSym<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 853 def : TerRMWPatExternalSym<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 854 855 def : TerRMWPatExternalSym<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 856 def : TerRMWPatExternalSym<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 857 def : TerRMWPatExternalSym<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 858 def : TerRMWPatExternalSym<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 859 860 // Truncating-extending ternary RMWs with just a constant offset 861 def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 862 def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 863 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 864 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 865 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 866 867 def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 868 def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 869 def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 870 def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 871 872 def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 873 def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 874 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 875 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 876 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 877 878 def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 879 def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 880 def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 881 def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 882 883 def : TerRMWPatExternSymOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 884 def : TerRMWPatExternSymOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 885 def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 886 def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 887 def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 888 889 def : TerRMWPatExternSymOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 890 def : TerRMWPatExternSymOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 891 def : TerRMWPatExternSymOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 892 def : TerRMWPatExternSymOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 893} 894 895let Predicates = [HasAtomics] in { 896defm : TerRMWTruncExtPattern< 897 atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, 898 ATOMIC_RMW8_U_CMPXCHG_I32, ATOMIC_RMW16_U_CMPXCHG_I32, 899 ATOMIC_RMW8_U_CMPXCHG_I64, ATOMIC_RMW16_U_CMPXCHG_I64, 900 ATOMIC_RMW32_U_CMPXCHG_I64>; 901} 902 903//===----------------------------------------------------------------------===// 904// Atomic wait / notify 905//===----------------------------------------------------------------------===// 906 907let Defs = [ARGUMENTS] in { 908let hasSideEffects = 1 in { 909defm ATOMIC_NOTIFY : 910 I<(outs I64:$dst), 911 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$count), 912 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 913 "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 914 "atomic.notify \t${off}, ${p2align}", 0xfe00>; 915let mayLoad = 1 in { 916defm ATOMIC_WAIT_I32 : 917 I<(outs I32:$dst), 918 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, I64:$timeout), 919 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 920 "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 921 "i32.atomic.wait \t${off}, ${p2align}", 0xfe01>; 922defm ATOMIC_WAIT_I64 : 923 I<(outs I32:$dst), 924 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, I64:$timeout), 925 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 926 "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 927 "i64.atomic.wait \t${off}, ${p2align}", 0xfe02>; 928} // mayLoad = 1 929} // hasSideEffects = 1 930} // Defs = [ARGUMENTS] 931 932let Predicates = [HasAtomics] in { 933// Select notifys with no constant offset. 934class NotifyPatNoOffset<Intrinsic kind> : 935 Pat<(i64 (kind I32:$addr, I64:$count)), 936 (ATOMIC_NOTIFY 0, 0, I32:$addr, I64:$count)>; 937def : NotifyPatNoOffset<int_wasm_atomic_notify>; 938 939// Select notifys with a constant offset. 940 941// Pattern with address + immediate offset 942class NotifyPatImmOff<Intrinsic kind, PatFrag operand> : 943 Pat<(i64 (kind (operand I32:$addr, imm:$off), I64:$count)), 944 (ATOMIC_NOTIFY 0, imm:$off, I32:$addr, I64:$count)>; 945def : NotifyPatImmOff<int_wasm_atomic_notify, regPlusImm>; 946def : NotifyPatImmOff<int_wasm_atomic_notify, or_is_add>; 947 948class NotifyPatGlobalAddr<Intrinsic kind> : 949 Pat<(i64 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 950 I64:$count)), 951 (ATOMIC_NOTIFY 0, tglobaladdr:$off, I32:$addr, I64:$count)>; 952def : NotifyPatGlobalAddr<int_wasm_atomic_notify>; 953 954class NotifyPatExternalSym<Intrinsic kind> : 955 Pat<(i64 (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), 956 I64:$count)), 957 (ATOMIC_NOTIFY 0, texternalsym:$off, I32:$addr, I64:$count)>; 958def : NotifyPatExternalSym<int_wasm_atomic_notify>; 959 960// Select notifys with just a constant offset. 961class NotifyPatOffsetOnly<Intrinsic kind> : 962 Pat<(i64 (kind imm:$off, I64:$count)), 963 (ATOMIC_NOTIFY 0, imm:$off, (CONST_I32 0), I64:$count)>; 964def : NotifyPatOffsetOnly<int_wasm_atomic_notify>; 965 966class NotifyPatGlobalAddrOffOnly<Intrinsic kind> : 967 Pat<(i64 (kind (WebAssemblywrapper tglobaladdr:$off), I64:$count)), 968 (ATOMIC_NOTIFY 0, tglobaladdr:$off, (CONST_I32 0), I64:$count)>; 969def : NotifyPatGlobalAddrOffOnly<int_wasm_atomic_notify>; 970 971class NotifyPatExternSymOffOnly<Intrinsic kind> : 972 Pat<(i64 (kind (WebAssemblywrapper texternalsym:$off), I64:$count)), 973 (ATOMIC_NOTIFY 0, texternalsym:$off, (CONST_I32 0), I64:$count)>; 974def : NotifyPatExternSymOffOnly<int_wasm_atomic_notify>; 975 976// Select waits with no constant offset. 977class WaitPatNoOffset<ValueType ty, Intrinsic kind, NI inst> : 978 Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), 979 (inst 0, 0, I32:$addr, ty:$exp, I64:$timeout)>; 980def : WaitPatNoOffset<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 981def : WaitPatNoOffset<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 982 983// Select waits with a constant offset. 984 985// Pattern with address + immediate offset 986class WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, NI inst> : 987 Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), 988 (inst 0, imm:$off, I32:$addr, ty:$exp, I64:$timeout)>; 989def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, regPlusImm, ATOMIC_WAIT_I32>; 990def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, or_is_add, ATOMIC_WAIT_I32>; 991def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, regPlusImm, ATOMIC_WAIT_I64>; 992def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, or_is_add, ATOMIC_WAIT_I64>; 993 994class WaitPatGlobalAddr<ValueType ty, Intrinsic kind, NI inst> : 995 Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 996 ty:$exp, I64:$timeout)), 997 (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, I64:$timeout)>; 998def : WaitPatGlobalAddr<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 999def : WaitPatGlobalAddr<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1000 1001class WaitPatExternalSym<ValueType ty, Intrinsic kind, NI inst> : 1002 Pat<(i32 (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), 1003 ty:$exp, I64:$timeout)), 1004 (inst 0, texternalsym:$off, I32:$addr, ty:$exp, I64:$timeout)>; 1005def : WaitPatExternalSym<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1006def : WaitPatExternalSym<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1007 1008// Select wait_i32, ATOMIC_WAIT_I32s with just a constant offset. 1009class WaitPatOffsetOnly<ValueType ty, Intrinsic kind, NI inst> : 1010 Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 1011 (inst 0, imm:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 1012def : WaitPatOffsetOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1013def : WaitPatOffsetOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1014 1015class WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, NI inst> : 1016 Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, I64:$timeout)), 1017 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 1018def : WaitPatGlobalAddrOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1019def : WaitPatGlobalAddrOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1020 1021class WaitPatExternSymOffOnly<ValueType ty, Intrinsic kind, NI inst> : 1022 Pat<(i32 (kind (WebAssemblywrapper texternalsym:$off), ty:$exp, 1023 I64:$timeout)), 1024 (inst 0, texternalsym:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 1025def : WaitPatExternSymOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1026def : WaitPatExternSymOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1027} // Predicates = [HasAtomics] 1028