1// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9/// 10/// \file 11/// WebAssembly Atomic operand code-gen constructs. 12/// 13//===----------------------------------------------------------------------===// 14 15//===----------------------------------------------------------------------===// 16// Atomic loads 17//===----------------------------------------------------------------------===// 18 19multiclass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 20 list<dag> pattern_r, string asmstr_r = "", 21 string asmstr_s = "", bits<32> inst = -1> { 22 defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 23 inst>, 24 Requires<[HasAtomics]>; 25} 26 27defm ATOMIC_LOAD_I32 : WebAssemblyLoad<I32, "i32.atomic.load", 0xfe10>; 28defm ATOMIC_LOAD_I64 : WebAssemblyLoad<I64, "i64.atomic.load", 0xfe11>; 29 30// Select loads with no constant offset. 31let Predicates = [HasAtomics] in { 32def : LoadPatNoOffset<i32, atomic_load_32, ATOMIC_LOAD_I32>; 33def : LoadPatNoOffset<i64, atomic_load_64, ATOMIC_LOAD_I64>; 34 35// Select loads with a constant offset. 36 37// Pattern with address + immediate offset 38def : LoadPatImmOff<i32, atomic_load_32, regPlusImm, ATOMIC_LOAD_I32>; 39def : LoadPatImmOff<i64, atomic_load_64, regPlusImm, ATOMIC_LOAD_I64>; 40def : LoadPatImmOff<i32, atomic_load_32, or_is_add, ATOMIC_LOAD_I32>; 41def : LoadPatImmOff<i64, atomic_load_64, or_is_add, ATOMIC_LOAD_I64>; 42 43def : LoadPatGlobalAddr<i32, atomic_load_32, ATOMIC_LOAD_I32>; 44def : LoadPatGlobalAddr<i64, atomic_load_64, ATOMIC_LOAD_I64>; 45 46def : LoadPatExternalSym<i32, atomic_load_32, ATOMIC_LOAD_I32>; 47def : LoadPatExternalSym<i64, atomic_load_64, ATOMIC_LOAD_I64>; 48 49// Select loads with just a constant offset. 50def : LoadPatOffsetOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 51def : LoadPatOffsetOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 52 53def : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 54def : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 55 56def : LoadPatExternSymOffOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 57def : LoadPatExternSymOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 58 59} // Predicates = [HasAtomics] 60 61// Extending loads. Note that there are only zero-extending atomic loads, no 62// sign-extending loads. 63defm ATOMIC_LOAD8_U_I32 : WebAssemblyLoad<I32, "i32.atomic.load8_u", 0xfe12>; 64defm ATOMIC_LOAD16_U_I32 : WebAssemblyLoad<I32, "i32.atomic.load16_u", 0xfe13>; 65defm ATOMIC_LOAD8_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load8_u", 0xfe14>; 66defm ATOMIC_LOAD16_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load16_u", 0xfe15>; 67defm ATOMIC_LOAD32_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load32_u", 0xfe16>; 68 69// Fragments for extending loads. These are different from regular loads because 70// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 71// therefore don't have the extension type field. So instead of matching that, 72// we match the patterns that the type legalizer expands them to. 73 74// We directly match zext patterns and select the zext atomic loads. 75// i32 (zext (i8 (atomic_load_8))) gets legalized to 76// i32 (and (i32 (atomic_load_8)), 255) 77// These can be selected to a single zero-extending atomic load instruction. 78def zext_aload_8_32 : 79 PatFrag<(ops node:$addr), (and (i32 (atomic_load_8 node:$addr)), 255)>; 80def zext_aload_16_32 : 81 PatFrag<(ops node:$addr), (and (i32 (atomic_load_16 node:$addr)), 65535)>; 82// Unlike regular loads, extension to i64 is handled differently than i32. 83// i64 (zext (i8 (atomic_load_8))) gets legalized to 84// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 85def zext_aload_8_64 : 86 PatFrag<(ops node:$addr), 87 (and (i64 (anyext (i32 (atomic_load_8 node:$addr)))), 255)>; 88def zext_aload_16_64 : 89 PatFrag<(ops node:$addr), 90 (and (i64 (anyext (i32 (atomic_load_16 node:$addr)))), 65535)>; 91def zext_aload_32_64 : 92 PatFrag<(ops node:$addr), 93 (zext (i32 (atomic_load node:$addr)))>; 94 95// We don't have single sext atomic load instructions. So for sext loads, we 96// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 97// results) and select a zext load; the next instruction will be sext_inreg 98// which is selected by itself. 99def sext_aload_8_64 : 100 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 101def sext_aload_16_64 : 102 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 103 104let Predicates = [HasAtomics] in { 105// Select zero-extending loads with no constant offset. 106def : LoadPatNoOffset<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 107def : LoadPatNoOffset<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 108def : LoadPatNoOffset<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 109def : LoadPatNoOffset<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 110def : LoadPatNoOffset<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 111 112// Select sign-extending loads with no constant offset 113def : LoadPatNoOffset<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 114def : LoadPatNoOffset<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 115def : LoadPatNoOffset<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 116def : LoadPatNoOffset<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 117// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s 118 119// Zero-extending loads with constant offset 120def : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, ATOMIC_LOAD8_U_I32>; 121def : LoadPatImmOff<i32, zext_aload_16_32, regPlusImm, ATOMIC_LOAD16_U_I32>; 122def : LoadPatImmOff<i32, zext_aload_8_32, or_is_add, ATOMIC_LOAD8_U_I32>; 123def : LoadPatImmOff<i32, zext_aload_16_32, or_is_add, ATOMIC_LOAD16_U_I32>; 124def : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 125def : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 126def : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, ATOMIC_LOAD32_U_I64>; 127def : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 128def : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 129def : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, ATOMIC_LOAD32_U_I64>; 130 131// Sign-extending loads with constant offset 132def : LoadPatImmOff<i32, atomic_load_8, regPlusImm, ATOMIC_LOAD8_U_I32>; 133def : LoadPatImmOff<i32, atomic_load_16, regPlusImm, ATOMIC_LOAD16_U_I32>; 134def : LoadPatImmOff<i32, atomic_load_8, or_is_add, ATOMIC_LOAD8_U_I32>; 135def : LoadPatImmOff<i32, atomic_load_16, or_is_add, ATOMIC_LOAD16_U_I32>; 136def : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 137def : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 138def : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 139def : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 140// No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 141 142def : LoadPatGlobalAddr<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 143def : LoadPatGlobalAddr<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 144def : LoadPatGlobalAddr<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 145def : LoadPatGlobalAddr<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 146def : LoadPatGlobalAddr<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 147def : LoadPatGlobalAddr<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 148def : LoadPatGlobalAddr<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 149def : LoadPatGlobalAddr<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 150def : LoadPatGlobalAddr<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 151 152def : LoadPatExternalSym<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 153def : LoadPatExternalSym<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 154def : LoadPatExternalSym<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 155def : LoadPatExternalSym<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 156def : LoadPatExternalSym<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 157def : LoadPatExternalSym<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 158def : LoadPatExternalSym<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 159def : LoadPatExternalSym<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 160def : LoadPatExternalSym<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 161 162// Extending loads with just a constant offset 163def : LoadPatOffsetOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 164def : LoadPatOffsetOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 165def : LoadPatOffsetOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 166def : LoadPatOffsetOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 167def : LoadPatOffsetOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 168def : LoadPatOffsetOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 169def : LoadPatOffsetOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 170def : LoadPatOffsetOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 171def : LoadPatOffsetOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 172 173def : LoadPatGlobalAddrOffOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 174def : LoadPatGlobalAddrOffOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 175def : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 176def : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 177def : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 178def : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 179def : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 180def : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 181def : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 182 183def : LoadPatExternSymOffOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 184def : LoadPatExternSymOffOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 185def : LoadPatExternSymOffOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 186def : LoadPatExternSymOffOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 187def : LoadPatExternSymOffOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 188def : LoadPatExternSymOffOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 189def : LoadPatExternSymOffOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 190def : LoadPatExternSymOffOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 191def : LoadPatExternSymOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 192 193} // Predicates = [HasAtomics] 194 195//===----------------------------------------------------------------------===// 196// Atomic stores 197//===----------------------------------------------------------------------===// 198 199defm ATOMIC_STORE_I32 : WebAssemblyStore<I32, "i32.atomic.store", 0xfe17>; 200defm ATOMIC_STORE_I64 : WebAssemblyStore<I64, "i64.atomic.store", 0xfe18>; 201 202// We need an 'atomic' version of store patterns because store and atomic_store 203// nodes have different operand orders: 204// store: (store $val, $ptr) 205// atomic_store: (store $ptr, $val) 206 207let Predicates = [HasAtomics] in { 208 209// Select stores with no constant offset. 210class AStorePatNoOffset<ValueType ty, PatFrag kind, NI inst> : 211 Pat<(kind I32:$addr, ty:$val), (inst 0, 0, I32:$addr, ty:$val)>; 212def : AStorePatNoOffset<i32, atomic_store_32, ATOMIC_STORE_I32>; 213def : AStorePatNoOffset<i64, atomic_store_64, ATOMIC_STORE_I64>; 214 215// Select stores with a constant offset. 216 217// Pattern with address + immediate offset 218class AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 219 Pat<(kind (operand I32:$addr, imm:$off), ty:$val), 220 (inst 0, imm:$off, I32:$addr, ty:$val)>; 221def : AStorePatImmOff<i32, atomic_store_32, regPlusImm, ATOMIC_STORE_I32>; 222def : AStorePatImmOff<i64, atomic_store_64, regPlusImm, ATOMIC_STORE_I64>; 223def : AStorePatImmOff<i32, atomic_store_32, or_is_add, ATOMIC_STORE_I32>; 224def : AStorePatImmOff<i64, atomic_store_64, or_is_add, ATOMIC_STORE_I64>; 225 226class AStorePatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 227 Pat<(kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 228 ty:$val), 229 (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>; 230def : AStorePatGlobalAddr<i32, atomic_store_32, ATOMIC_STORE_I32>; 231def : AStorePatGlobalAddr<i64, atomic_store_64, ATOMIC_STORE_I64>; 232 233class AStorePatExternalSym<ValueType ty, PatFrag kind, NI inst> : 234 Pat<(kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), ty:$val), 235 (inst 0, texternalsym:$off, I32:$addr, ty:$val)>; 236def : AStorePatExternalSym<i32, atomic_store_32, ATOMIC_STORE_I32>; 237def : AStorePatExternalSym<i64, atomic_store_64, ATOMIC_STORE_I64>; 238 239// Select stores with just a constant offset. 240class AStorePatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 241 Pat<(kind imm:$off, ty:$val), (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 242def : AStorePatOffsetOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 243def : AStorePatOffsetOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 244 245class AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 246 Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 247 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 248def : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 249def : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 250 251class AStorePatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> : 252 Pat<(kind (WebAssemblywrapper texternalsym:$off), ty:$val), 253 (inst 0, texternalsym:$off, (CONST_I32 0), ty:$val)>; 254def : AStorePatExternSymOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 255def : AStorePatExternSymOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 256 257} // Predicates = [HasAtomics] 258 259// Truncating stores. 260defm ATOMIC_STORE8_I32 : WebAssemblyStore<I32, "i32.atomic.store8", 0xfe19>; 261defm ATOMIC_STORE16_I32 : WebAssemblyStore<I32, "i32.atomic.store16", 0xfe1a>; 262defm ATOMIC_STORE8_I64 : WebAssemblyStore<I64, "i64.atomic.store8", 0xfe1b>; 263defm ATOMIC_STORE16_I64 : WebAssemblyStore<I64, "i64.atomic.store16", 0xfe1c>; 264defm ATOMIC_STORE32_I64 : WebAssemblyStore<I64, "i64.atomic.store32", 0xfe1d>; 265 266// Fragments for truncating stores. 267 268// We don't have single truncating atomic store instructions. For 32-bit 269// instructions, we just need to match bare atomic stores. On the other hand, 270// truncating stores from i64 values are once truncated to i32 first. 271class trunc_astore_64<PatFrag kind> : 272 PatFrag<(ops node:$addr, node:$val), 273 (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 274def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 275def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 276def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 277 278let Predicates = [HasAtomics] in { 279 280// Truncating stores with no constant offset 281def : AStorePatNoOffset<i32, atomic_store_8, ATOMIC_STORE8_I32>; 282def : AStorePatNoOffset<i32, atomic_store_16, ATOMIC_STORE16_I32>; 283def : AStorePatNoOffset<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 284def : AStorePatNoOffset<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 285def : AStorePatNoOffset<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 286 287// Truncating stores with a constant offset 288def : AStorePatImmOff<i32, atomic_store_8, regPlusImm, ATOMIC_STORE8_I32>; 289def : AStorePatImmOff<i32, atomic_store_16, regPlusImm, ATOMIC_STORE16_I32>; 290def : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, ATOMIC_STORE8_I64>; 291def : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, ATOMIC_STORE16_I64>; 292def : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, ATOMIC_STORE32_I64>; 293def : AStorePatImmOff<i32, atomic_store_8, or_is_add, ATOMIC_STORE8_I32>; 294def : AStorePatImmOff<i32, atomic_store_16, or_is_add, ATOMIC_STORE16_I32>; 295def : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, ATOMIC_STORE8_I64>; 296def : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, ATOMIC_STORE16_I64>; 297def : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, ATOMIC_STORE32_I64>; 298 299def : AStorePatGlobalAddr<i32, atomic_store_8, ATOMIC_STORE8_I32>; 300def : AStorePatGlobalAddr<i32, atomic_store_16, ATOMIC_STORE16_I32>; 301def : AStorePatGlobalAddr<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 302def : AStorePatGlobalAddr<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 303def : AStorePatGlobalAddr<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 304 305def : AStorePatExternalSym<i32, atomic_store_8, ATOMIC_STORE8_I32>; 306def : AStorePatExternalSym<i32, atomic_store_16, ATOMIC_STORE16_I32>; 307def : AStorePatExternalSym<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 308def : AStorePatExternalSym<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 309def : AStorePatExternalSym<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 310 311// Truncating stores with just a constant offset 312def : AStorePatOffsetOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 313def : AStorePatOffsetOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 314def : AStorePatOffsetOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 315def : AStorePatOffsetOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 316def : AStorePatOffsetOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 317 318def : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 319def : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 320def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 321def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 322def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 323 324def : AStorePatExternSymOffOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 325def : AStorePatExternSymOffOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 326def : AStorePatExternSymOffOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 327def : AStorePatExternSymOffOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 328def : AStorePatExternSymOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 329 330} // Predicates = [HasAtomics] 331 332//===----------------------------------------------------------------------===// 333// Atomic binary read-modify-writes 334//===----------------------------------------------------------------------===// 335 336multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string Name, int Opcode> { 337 defm "" : I<(outs rc:$dst), 338 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 339 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 340 !strconcat(Name, "\t$dst, ${off}(${addr})${p2align}, $val"), 341 !strconcat(Name, "\t${off}, ${p2align}"), Opcode>; 342} 343 344defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0xfe1e>; 345defm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0xfe1f>; 346defm ATOMIC_RMW8_U_ADD_I32 : 347 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0xfe20>; 348defm ATOMIC_RMW16_U_ADD_I32 : 349 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0xfe21>; 350defm ATOMIC_RMW8_U_ADD_I64 : 351 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0xfe22>; 352defm ATOMIC_RMW16_U_ADD_I64 : 353 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0xfe23>; 354defm ATOMIC_RMW32_U_ADD_I64 : 355 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0xfe24>; 356 357defm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0xfe25>; 358defm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0xfe26>; 359defm ATOMIC_RMW8_U_SUB_I32 : 360 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0xfe27>; 361defm ATOMIC_RMW16_U_SUB_I32 : 362 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0xfe28>; 363defm ATOMIC_RMW8_U_SUB_I64 : 364 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0xfe29>; 365defm ATOMIC_RMW16_U_SUB_I64 : 366 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0xfe2a>; 367defm ATOMIC_RMW32_U_SUB_I64 : 368 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0xfe2b>; 369 370defm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0xfe2c>; 371defm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0xfe2d>; 372defm ATOMIC_RMW8_U_AND_I32 : 373 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0xfe2e>; 374defm ATOMIC_RMW16_U_AND_I32 : 375 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0xfe2f>; 376defm ATOMIC_RMW8_U_AND_I64 : 377 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0xfe30>; 378defm ATOMIC_RMW16_U_AND_I64 : 379 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0xfe31>; 380defm ATOMIC_RMW32_U_AND_I64 : 381 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0xfe32>; 382 383defm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0xfe33>; 384defm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0xfe34>; 385defm ATOMIC_RMW8_U_OR_I32 : 386 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0xfe35>; 387defm ATOMIC_RMW16_U_OR_I32 : 388 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0xfe36>; 389defm ATOMIC_RMW8_U_OR_I64 : 390 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0xfe37>; 391defm ATOMIC_RMW16_U_OR_I64 : 392 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0xfe38>; 393defm ATOMIC_RMW32_U_OR_I64 : 394 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0xfe39>; 395 396defm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0xfe3a>; 397defm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0xfe3b>; 398defm ATOMIC_RMW8_U_XOR_I32 : 399 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0xfe3c>; 400defm ATOMIC_RMW16_U_XOR_I32 : 401 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0xfe3d>; 402defm ATOMIC_RMW8_U_XOR_I64 : 403 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0xfe3e>; 404defm ATOMIC_RMW16_U_XOR_I64 : 405 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0xfe3f>; 406defm ATOMIC_RMW32_U_XOR_I64 : 407 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0xfe40>; 408 409defm ATOMIC_RMW_XCHG_I32 : 410 WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0xfe41>; 411defm ATOMIC_RMW_XCHG_I64 : 412 WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0xfe42>; 413defm ATOMIC_RMW8_U_XCHG_I32 : 414 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0xfe43>; 415defm ATOMIC_RMW16_U_XCHG_I32 : 416 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0xfe44>; 417defm ATOMIC_RMW8_U_XCHG_I64 : 418 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0xfe45>; 419defm ATOMIC_RMW16_U_XCHG_I64 : 420 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0xfe46>; 421defm ATOMIC_RMW32_U_XCHG_I64 : 422 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0xfe47>; 423 424// Select binary RMWs with no constant offset. 425class BinRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 426 Pat<(ty (kind I32:$addr, ty:$val)), (inst 0, 0, I32:$addr, ty:$val)>; 427 428// Select binary RMWs with a constant offset. 429 430// Pattern with address + immediate offset 431class BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 432 Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), 433 (inst 0, imm:$off, I32:$addr, ty:$val)>; 434 435class BinRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 436 Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 437 ty:$val)), 438 (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>; 439 440class BinRMWPatExternalSym<ValueType ty, PatFrag kind, NI inst> : 441 Pat<(ty (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), 442 ty:$val)), 443 (inst 0, texternalsym:$off, I32:$addr, ty:$val)>; 444 445// Select binary RMWs with just a constant offset. 446class BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 447 Pat<(ty (kind imm:$off, ty:$val)), 448 (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 449 450class BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 451 Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 452 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 453 454class BinRMWPatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> : 455 Pat<(ty (kind (WebAssemblywrapper texternalsym:$off), ty:$val)), 456 (inst 0, texternalsym:$off, (CONST_I32 0), ty:$val)>; 457 458// Patterns for various addressing modes. 459multiclass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 460 NI inst_64> { 461 def : BinRMWPatNoOffset<i32, rmw_32, inst_32>; 462 def : BinRMWPatNoOffset<i64, rmw_64, inst_64>; 463 464 def : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 465 def : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 466 def : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 467 def : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 468 469 def : BinRMWPatGlobalAddr<i32, rmw_32, inst_32>; 470 def : BinRMWPatGlobalAddr<i64, rmw_64, inst_64>; 471 472 def : BinRMWPatExternalSym<i32, rmw_32, inst_32>; 473 def : BinRMWPatExternalSym<i64, rmw_64, inst_64>; 474 475 def : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>; 476 def : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>; 477 478 def : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 479 def : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 480 481 def : BinRMWPatExternSymOffOnly<i32, rmw_32, inst_32>; 482 def : BinRMWPatExternSymOffOnly<i64, rmw_64, inst_64>; 483} 484 485let Predicates = [HasAtomics] in { 486defm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, ATOMIC_RMW_ADD_I32, 487 ATOMIC_RMW_ADD_I64>; 488defm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, ATOMIC_RMW_SUB_I32, 489 ATOMIC_RMW_SUB_I64>; 490defm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, ATOMIC_RMW_AND_I32, 491 ATOMIC_RMW_AND_I64>; 492defm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, ATOMIC_RMW_OR_I32, 493 ATOMIC_RMW_OR_I64>; 494defm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, ATOMIC_RMW_XOR_I32, 495 ATOMIC_RMW_XOR_I64>; 496defm : BinRMWPattern<atomic_swap_32, atomic_swap_64, ATOMIC_RMW_XCHG_I32, 497 ATOMIC_RMW_XCHG_I64>; 498} // Predicates = [HasAtomics] 499 500// Truncating & zero-extending binary RMW patterns. 501// These are combined patterns of truncating store patterns and zero-extending 502// load patterns above. 503class zext_bin_rmw_8_32<PatFrag kind> : 504 PatFrag<(ops node:$addr, node:$val), 505 (and (i32 (kind node:$addr, node:$val)), 255)>; 506class zext_bin_rmw_16_32<PatFrag kind> : 507 PatFrag<(ops node:$addr, node:$val), 508 (and (i32 (kind node:$addr, node:$val)), 65535)>; 509class zext_bin_rmw_8_64<PatFrag kind> : 510 PatFrag<(ops node:$addr, node:$val), 511 (and (i64 (anyext (i32 (kind node:$addr, 512 (i32 (trunc (i64 node:$val))))))), 255)>; 513class zext_bin_rmw_16_64<PatFrag kind> : 514 PatFrag<(ops node:$addr, node:$val), 515 (and (i64 (anyext (i32 (kind node:$addr, 516 (i32 (trunc (i64 node:$val))))))), 65535)>; 517class zext_bin_rmw_32_64<PatFrag kind> : 518 PatFrag<(ops node:$addr, node:$val), 519 (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 520 521// Truncating & sign-extending binary RMW patterns. 522// These are combined patterns of truncating store patterns and sign-extending 523// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 524// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 525// is selected by itself. 526class sext_bin_rmw_8_32<PatFrag kind> : 527 PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 528class sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 529class sext_bin_rmw_8_64<PatFrag kind> : 530 PatFrag<(ops node:$addr, node:$val), 531 (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 532class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 533// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 534 535// Patterns for various addressing modes for truncating-extending binary RMWs. 536multiclass BinRMWTruncExtPattern< 537 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 538 NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 539 // Truncating-extending binary RMWs with no constant offset 540 def : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 541 def : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 542 def : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 543 def : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 544 def : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 545 546 def : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 547 def : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 548 def : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 549 def : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 550 551 // Truncating-extending binary RMWs with a constant offset 552 def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 553 def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 554 def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 555 def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 556 def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 557 def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 558 def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 559 def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 560 def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 561 def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 562 563 def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 564 def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 565 def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 566 def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 567 def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 568 def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 569 def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 570 def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 571 572 def : BinRMWPatGlobalAddr<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 573 def : BinRMWPatGlobalAddr<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 574 def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 575 def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 576 def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 577 578 def : BinRMWPatGlobalAddr<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 579 def : BinRMWPatGlobalAddr<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 580 def : BinRMWPatGlobalAddr<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 581 def : BinRMWPatGlobalAddr<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 582 583 def : BinRMWPatExternalSym<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 584 def : BinRMWPatExternalSym<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 585 def : BinRMWPatExternalSym<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 586 def : BinRMWPatExternalSym<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 587 def : BinRMWPatExternalSym<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 588 589 def : BinRMWPatExternalSym<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 590 def : BinRMWPatExternalSym<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 591 def : BinRMWPatExternalSym<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 592 def : BinRMWPatExternalSym<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 593 594 // Truncating-extending binary RMWs with just a constant offset 595 def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 596 def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 597 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 598 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 599 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 600 601 def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 602 def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 603 def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 604 def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 605 606 def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 607 def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 608 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 609 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 610 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 611 612 def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 613 def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 614 def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 615 def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 616 617 def : BinRMWPatExternSymOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 618 def : BinRMWPatExternSymOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 619 def : BinRMWPatExternSymOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 620 def : BinRMWPatExternSymOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 621 def : BinRMWPatExternSymOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 622 623 def : BinRMWPatExternSymOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 624 def : BinRMWPatExternSymOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 625 def : BinRMWPatExternSymOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 626 def : BinRMWPatExternSymOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 627} 628 629let Predicates = [HasAtomics] in { 630defm : BinRMWTruncExtPattern< 631 atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, 632 ATOMIC_RMW8_U_ADD_I32, ATOMIC_RMW16_U_ADD_I32, 633 ATOMIC_RMW8_U_ADD_I64, ATOMIC_RMW16_U_ADD_I64, ATOMIC_RMW32_U_ADD_I64>; 634defm : BinRMWTruncExtPattern< 635 atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, 636 ATOMIC_RMW8_U_SUB_I32, ATOMIC_RMW16_U_SUB_I32, 637 ATOMIC_RMW8_U_SUB_I64, ATOMIC_RMW16_U_SUB_I64, ATOMIC_RMW32_U_SUB_I64>; 638defm : BinRMWTruncExtPattern< 639 atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, 640 ATOMIC_RMW8_U_AND_I32, ATOMIC_RMW16_U_AND_I32, 641 ATOMIC_RMW8_U_AND_I64, ATOMIC_RMW16_U_AND_I64, ATOMIC_RMW32_U_AND_I64>; 642defm : BinRMWTruncExtPattern< 643 atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, 644 ATOMIC_RMW8_U_OR_I32, ATOMIC_RMW16_U_OR_I32, 645 ATOMIC_RMW8_U_OR_I64, ATOMIC_RMW16_U_OR_I64, ATOMIC_RMW32_U_OR_I64>; 646defm : BinRMWTruncExtPattern< 647 atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, 648 ATOMIC_RMW8_U_XOR_I32, ATOMIC_RMW16_U_XOR_I32, 649 ATOMIC_RMW8_U_XOR_I64, ATOMIC_RMW16_U_XOR_I64, ATOMIC_RMW32_U_XOR_I64>; 650defm : BinRMWTruncExtPattern< 651 atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, 652 ATOMIC_RMW8_U_XCHG_I32, ATOMIC_RMW16_U_XCHG_I32, 653 ATOMIC_RMW8_U_XCHG_I64, ATOMIC_RMW16_U_XCHG_I64, ATOMIC_RMW32_U_XCHG_I64>; 654} // Predicates = [HasAtomics] 655 656//===----------------------------------------------------------------------===// 657// Atomic ternary read-modify-writes 658//===----------------------------------------------------------------------===// 659 660// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 661// flag}. When we use the success flag or both values, we can't make use of i64 662// truncate/extend versions of instructions for now, which is suboptimal. 663// Consider adding a pass after instruction selection that optimizes this case 664// if it is frequent. 665 666multiclass WebAssemblyTerRMW<WebAssemblyRegClass rc, string Name, int Opcode> { 667 defm "" : I<(outs rc:$dst), 668 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 669 rc:$new), 670 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 671 !strconcat(Name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new"), 672 !strconcat(Name, "\t${off}, ${p2align}"), Opcode>; 673} 674 675defm ATOMIC_RMW_CMPXCHG_I32 : 676 WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0xfe48>; 677defm ATOMIC_RMW_CMPXCHG_I64 : 678 WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0xfe49>; 679defm ATOMIC_RMW8_U_CMPXCHG_I32 : 680 WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0xfe4a>; 681defm ATOMIC_RMW16_U_CMPXCHG_I32 : 682 WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0xfe4b>; 683defm ATOMIC_RMW8_U_CMPXCHG_I64 : 684 WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0xfe4c>; 685defm ATOMIC_RMW16_U_CMPXCHG_I64 : 686 WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0xfe4d>; 687defm ATOMIC_RMW32_U_CMPXCHG_I64 : 688 WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0xfe4e>; 689 690// Select ternary RMWs with no constant offset. 691class TerRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 692 Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), 693 (inst 0, 0, I32:$addr, ty:$exp, ty:$new)>; 694 695// Select ternary RMWs with a constant offset. 696 697// Pattern with address + immediate offset 698class TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 699 Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), 700 (inst 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>; 701 702class TerRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : 703 Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 704 ty:$exp, ty:$new)), 705 (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, ty:$new)>; 706 707class TerRMWPatExternalSym<ValueType ty, PatFrag kind, NI inst> : 708 Pat<(ty (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), 709 ty:$exp, ty:$new)), 710 (inst 0, texternalsym:$off, I32:$addr, ty:$exp, ty:$new)>; 711 712// Select ternary RMWs with just a constant offset. 713class TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 714 Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 715 (inst 0, imm:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 716 717class TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 718 Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 719 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 720 721class TerRMWPatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> : 722 Pat<(ty (kind (WebAssemblywrapper texternalsym:$off), ty:$exp, ty:$new)), 723 (inst 0, texternalsym:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 724 725// Patterns for various addressing modes. 726multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 727 NI inst_64> { 728 def : TerRMWPatNoOffset<i32, rmw_32, inst_32>; 729 def : TerRMWPatNoOffset<i64, rmw_64, inst_64>; 730 731 def : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 732 def : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 733 def : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 734 def : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 735 736 def : TerRMWPatGlobalAddr<i32, rmw_32, inst_32>; 737 def : TerRMWPatGlobalAddr<i64, rmw_64, inst_64>; 738 739 def : TerRMWPatExternalSym<i32, rmw_32, inst_32>; 740 def : TerRMWPatExternalSym<i64, rmw_64, inst_64>; 741 742 def : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; 743 def : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; 744 745 def : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 746 def : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 747 748 def : TerRMWPatExternSymOffOnly<i32, rmw_32, inst_32>; 749 def : TerRMWPatExternSymOffOnly<i64, rmw_64, inst_64>; 750} 751 752let Predicates = [HasAtomics] in { 753defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, 754 ATOMIC_RMW_CMPXCHG_I32, ATOMIC_RMW_CMPXCHG_I64>; 755} // Predicates = [HasAtomics] 756 757// Truncating & zero-extending ternary RMW patterns. 758// DAG legalization & optimization before instruction selection may introduce 759// additional nodes such as anyext or assertzext depending on operand types. 760class zext_ter_rmw_8_32<PatFrag kind> : 761 PatFrag<(ops node:$addr, node:$exp, node:$new), 762 (and (i32 (kind node:$addr, node:$exp, node:$new)), 255)>; 763class zext_ter_rmw_16_32<PatFrag kind> : 764 PatFrag<(ops node:$addr, node:$exp, node:$new), 765 (and (i32 (kind node:$addr, node:$exp, node:$new)), 65535)>; 766class zext_ter_rmw_8_64<PatFrag kind> : 767 PatFrag<(ops node:$addr, node:$exp, node:$new), 768 (zext (i32 (assertzext (i32 (kind node:$addr, 769 (i32 (trunc (i64 node:$exp))), 770 (i32 (trunc (i64 node:$new))))))))>; 771class zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 772class zext_ter_rmw_32_64<PatFrag kind> : 773 PatFrag<(ops node:$addr, node:$exp, node:$new), 774 (zext (i32 (kind node:$addr, 775 (i32 (trunc (i64 node:$exp))), 776 (i32 (trunc (i64 node:$new))))))>; 777 778// Truncating & sign-extending ternary RMW patterns. 779// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 780// zext RMW; the next instruction will be sext_inreg which is selected by 781// itself. 782class sext_ter_rmw_8_32<PatFrag kind> : 783 PatFrag<(ops node:$addr, node:$exp, node:$new), 784 (kind node:$addr, node:$exp, node:$new)>; 785class sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 786class sext_ter_rmw_8_64<PatFrag kind> : 787 PatFrag<(ops node:$addr, node:$exp, node:$new), 788 (anyext (i32 (assertzext (i32 789 (kind node:$addr, 790 (i32 (trunc (i64 node:$exp))), 791 (i32 (trunc (i64 node:$new))))))))>; 792class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 793// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 794 795// Patterns for various addressing modes for truncating-extending ternary RMWs. 796multiclass TerRMWTruncExtPattern< 797 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 798 NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 799 // Truncating-extending ternary RMWs with no constant offset 800 def : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 801 def : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 802 def : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 803 def : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 804 def : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 805 806 def : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 807 def : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 808 def : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 809 def : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 810 811 // Truncating-extending ternary RMWs with a constant offset 812 def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 813 def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 814 def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 815 def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 816 def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 817 def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 818 def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 819 def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 820 def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 821 def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 822 823 def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 824 def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 825 def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 826 def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 827 def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 828 def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 829 def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 830 def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 831 832 def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 833 def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 834 def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 835 def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 836 def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 837 838 def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 839 def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 840 def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 841 def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 842 843 def : TerRMWPatExternalSym<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 844 def : TerRMWPatExternalSym<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 845 def : TerRMWPatExternalSym<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 846 def : TerRMWPatExternalSym<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 847 def : TerRMWPatExternalSym<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 848 849 def : TerRMWPatExternalSym<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 850 def : TerRMWPatExternalSym<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 851 def : TerRMWPatExternalSym<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 852 def : TerRMWPatExternalSym<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 853 854 // Truncating-extending ternary RMWs with just a constant offset 855 def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 856 def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 857 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 858 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 859 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 860 861 def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 862 def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 863 def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 864 def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 865 866 def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 867 def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 868 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 869 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 870 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 871 872 def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 873 def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 874 def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 875 def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 876 877 def : TerRMWPatExternSymOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 878 def : TerRMWPatExternSymOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 879 def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 880 def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 881 def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 882 883 def : TerRMWPatExternSymOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 884 def : TerRMWPatExternSymOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 885 def : TerRMWPatExternSymOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 886 def : TerRMWPatExternSymOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 887} 888 889let Predicates = [HasAtomics] in { 890defm : TerRMWTruncExtPattern< 891 atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, 892 ATOMIC_RMW8_U_CMPXCHG_I32, ATOMIC_RMW16_U_CMPXCHG_I32, 893 ATOMIC_RMW8_U_CMPXCHG_I64, ATOMIC_RMW16_U_CMPXCHG_I64, 894 ATOMIC_RMW32_U_CMPXCHG_I64>; 895} 896 897//===----------------------------------------------------------------------===// 898// Atomic wait / notify 899//===----------------------------------------------------------------------===// 900 901let hasSideEffects = 1 in { 902defm ATOMIC_NOTIFY : 903 I<(outs I32:$dst), 904 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 905 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 906 "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 907 "atomic.notify \t${off}, ${p2align}", 0xfe00>; 908let mayLoad = 1 in { 909defm ATOMIC_WAIT_I32 : 910 I<(outs I32:$dst), 911 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, I64:$timeout), 912 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 913 "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 914 "i32.atomic.wait \t${off}, ${p2align}", 0xfe01>; 915defm ATOMIC_WAIT_I64 : 916 I<(outs I32:$dst), 917 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, I64:$timeout), 918 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 919 "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 920 "i64.atomic.wait \t${off}, ${p2align}", 0xfe02>; 921} // mayLoad = 1 922} // hasSideEffects = 1 923 924let Predicates = [HasAtomics] in { 925// Select notifys with no constant offset. 926class NotifyPatNoOffset<Intrinsic kind> : 927 Pat<(i32 (kind I32:$addr, I32:$count)), 928 (ATOMIC_NOTIFY 0, 0, I32:$addr, I32:$count)>; 929def : NotifyPatNoOffset<int_wasm_atomic_notify>; 930 931// Select notifys with a constant offset. 932 933// Pattern with address + immediate offset 934class NotifyPatImmOff<Intrinsic kind, PatFrag operand> : 935 Pat<(i32 (kind (operand I32:$addr, imm:$off), I32:$count)), 936 (ATOMIC_NOTIFY 0, imm:$off, I32:$addr, I32:$count)>; 937def : NotifyPatImmOff<int_wasm_atomic_notify, regPlusImm>; 938def : NotifyPatImmOff<int_wasm_atomic_notify, or_is_add>; 939 940class NotifyPatGlobalAddr<Intrinsic kind> : 941 Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 942 I32:$count)), 943 (ATOMIC_NOTIFY 0, tglobaladdr:$off, I32:$addr, I32:$count)>; 944def : NotifyPatGlobalAddr<int_wasm_atomic_notify>; 945 946class NotifyPatExternalSym<Intrinsic kind> : 947 Pat<(i32 (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), 948 I32:$count)), 949 (ATOMIC_NOTIFY 0, texternalsym:$off, I32:$addr, I32:$count)>; 950def : NotifyPatExternalSym<int_wasm_atomic_notify>; 951 952// Select notifys with just a constant offset. 953class NotifyPatOffsetOnly<Intrinsic kind> : 954 Pat<(i32 (kind imm:$off, I32:$count)), 955 (ATOMIC_NOTIFY 0, imm:$off, (CONST_I32 0), I32:$count)>; 956def : NotifyPatOffsetOnly<int_wasm_atomic_notify>; 957 958class NotifyPatGlobalAddrOffOnly<Intrinsic kind> : 959 Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), I32:$count)), 960 (ATOMIC_NOTIFY 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)>; 961def : NotifyPatGlobalAddrOffOnly<int_wasm_atomic_notify>; 962 963class NotifyPatExternSymOffOnly<Intrinsic kind> : 964 Pat<(i32 (kind (WebAssemblywrapper texternalsym:$off), I32:$count)), 965 (ATOMIC_NOTIFY 0, texternalsym:$off, (CONST_I32 0), I32:$count)>; 966def : NotifyPatExternSymOffOnly<int_wasm_atomic_notify>; 967 968// Select waits with no constant offset. 969class WaitPatNoOffset<ValueType ty, Intrinsic kind, NI inst> : 970 Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), 971 (inst 0, 0, I32:$addr, ty:$exp, I64:$timeout)>; 972def : WaitPatNoOffset<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 973def : WaitPatNoOffset<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 974 975// Select waits with a constant offset. 976 977// Pattern with address + immediate offset 978class WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, NI inst> : 979 Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), 980 (inst 0, imm:$off, I32:$addr, ty:$exp, I64:$timeout)>; 981def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, regPlusImm, ATOMIC_WAIT_I32>; 982def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, or_is_add, ATOMIC_WAIT_I32>; 983def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, regPlusImm, ATOMIC_WAIT_I64>; 984def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, or_is_add, ATOMIC_WAIT_I64>; 985 986class WaitPatGlobalAddr<ValueType ty, Intrinsic kind, NI inst> : 987 Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), 988 ty:$exp, I64:$timeout)), 989 (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, I64:$timeout)>; 990def : WaitPatGlobalAddr<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 991def : WaitPatGlobalAddr<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 992 993class WaitPatExternalSym<ValueType ty, Intrinsic kind, NI inst> : 994 Pat<(i32 (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), 995 ty:$exp, I64:$timeout)), 996 (inst 0, texternalsym:$off, I32:$addr, ty:$exp, I64:$timeout)>; 997def : WaitPatExternalSym<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 998def : WaitPatExternalSym<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 999 1000// Select wait_i32, ATOMIC_WAIT_I32s with just a constant offset. 1001class WaitPatOffsetOnly<ValueType ty, Intrinsic kind, NI inst> : 1002 Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 1003 (inst 0, imm:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 1004def : WaitPatOffsetOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1005def : WaitPatOffsetOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1006 1007class WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, NI inst> : 1008 Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, I64:$timeout)), 1009 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 1010def : WaitPatGlobalAddrOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1011def : WaitPatGlobalAddrOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1012 1013class WaitPatExternSymOffOnly<ValueType ty, Intrinsic kind, NI inst> : 1014 Pat<(i32 (kind (WebAssemblywrapper texternalsym:$off), ty:$exp, 1015 I64:$timeout)), 1016 (inst 0, texternalsym:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 1017def : WaitPatExternSymOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 1018def : WaitPatExternSymOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 1019} // Predicates = [HasAtomics] 1020