1// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8/// 9/// \file 10/// WebAssembly Atomic operand code-gen constructs. 11/// 12//===----------------------------------------------------------------------===// 13 14let UseNamedOperandTable = 1 in 15multiclass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 16 list<dag> pattern_r, string asmstr_r, 17 string asmstr_s, bits<32> atomic_op, 18 string is64 = "false"> { 19 defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 20 !or(0xfe00, !and(0xff, atomic_op)), is64>, 21 Requires<[HasAtomics]>; 22} 23 24multiclass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "", 25 bits<32> atomic_op = -1> { 26 defm "" : NRI<oops, iops, pattern, asmstr, 27 !or(0xfe00, !and(0xff, atomic_op))>, 28 Requires<[HasAtomics]>; 29} 30 31//===----------------------------------------------------------------------===// 32// Atomic wait / notify 33//===----------------------------------------------------------------------===// 34 35let hasSideEffects = 1 in { 36defm MEMORY_ATOMIC_NOTIFY_A32 : 37 ATOMIC_I<(outs I32:$dst), 38 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 39 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 40 "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 41 "memory.atomic.notify \t${off}${p2align}", 0x00, "false">; 42defm MEMORY_ATOMIC_NOTIFY_A64 : 43 ATOMIC_I<(outs I32:$dst), 44 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$count), 45 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 46 "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 47 "memory.atomic.notify \t${off}${p2align}", 0x00, "true">; 48let mayLoad = 1 in { 49defm MEMORY_ATOMIC_WAIT32_A32 : 50 ATOMIC_I<(outs I32:$dst), 51 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, 52 I64:$timeout), 53 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 54 "memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 55 "memory.atomic.wait32 \t${off}${p2align}", 0x01, "false">; 56defm MEMORY_ATOMIC_WAIT32_A64 : 57 ATOMIC_I<(outs I32:$dst), 58 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$exp, 59 I64:$timeout), 60 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 61 "memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 62 "memory.atomic.wait32 \t${off}${p2align}", 0x01, "true">; 63defm MEMORY_ATOMIC_WAIT64_A32 : 64 ATOMIC_I<(outs I32:$dst), 65 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, 66 I64:$timeout), 67 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 68 "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 69 "memory.atomic.wait64 \t${off}${p2align}", 0x02, "false">; 70defm MEMORY_ATOMIC_WAIT64_A64 : 71 ATOMIC_I<(outs I32:$dst), 72 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I64:$exp, 73 I64:$timeout), 74 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 75 "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 76 "memory.atomic.wait64 \t${off}${p2align}", 0x02, "true">; 77} // mayLoad = 1 78} // hasSideEffects = 1 79 80let Predicates = [HasAtomics] in { 81// Select notifys with no constant offset. 82def NotifyPatNoOffset_A32 : 83 Pat<(i32 (int_wasm_memory_atomic_notify I32:$addr, I32:$count)), 84 (MEMORY_ATOMIC_NOTIFY_A32 0, 0, I32:$addr, I32:$count)>, 85 Requires<[HasAddr32]>; 86def NotifyPatNoOffset_A64 : 87 Pat<(i32 (int_wasm_memory_atomic_notify I64:$addr, I32:$count)), 88 (MEMORY_ATOMIC_NOTIFY_A64 0, 0, I64:$addr, I32:$count)>, 89 Requires<[HasAddr64]>; 90 91// Select notifys with a constant offset. 92 93// Pattern with address + immediate offset 94multiclass NotifyPatImmOff<PatFrag operand, string inst> { 95 def : Pat<(i32 (int_wasm_memory_atomic_notify (operand I32:$addr, imm:$off), 96 I32:$count)), 97 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, I32:$count)>, 98 Requires<[HasAddr32]>; 99 def : Pat<(i32 (int_wasm_memory_atomic_notify (operand I64:$addr, imm:$off), 100 I32:$count)), 101 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, I32:$count)>, 102 Requires<[HasAddr64]>; 103} 104defm : NotifyPatImmOff<regPlusImm, "MEMORY_ATOMIC_NOTIFY">; 105defm : NotifyPatImmOff<or_is_add, "MEMORY_ATOMIC_NOTIFY">; 106 107// Select notifys with just a constant offset. 108def NotifyPatOffsetOnly_A32 : 109 Pat<(i32 (int_wasm_memory_atomic_notify imm:$off, I32:$count)), 110 (MEMORY_ATOMIC_NOTIFY_A32 0, imm:$off, (CONST_I32 0), I32:$count)>, 111 Requires<[HasAddr32]>; 112def NotifyPatOffsetOnly_A64 : 113 Pat<(i32 (int_wasm_memory_atomic_notify imm:$off, I32:$count)), 114 (MEMORY_ATOMIC_NOTIFY_A64 0, imm:$off, (CONST_I64 0), I32:$count)>, 115 Requires<[HasAddr64]>; 116 117def NotifyPatGlobalAddrOffOnly_A32 : 118 Pat<(i32 (int_wasm_memory_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 119 I32:$count)), 120 (MEMORY_ATOMIC_NOTIFY_A32 0, tglobaladdr:$off, (CONST_I32 0), I32:$count) 121 >, 122 Requires<[HasAddr32]>; 123def NotifyPatGlobalAddrOffOnly_A64 : 124 Pat<(i32 (int_wasm_memory_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 125 I32:$count)), 126 (MEMORY_ATOMIC_NOTIFY_A64 0, tglobaladdr:$off, (CONST_I64 0), I32:$count) 127 >, 128 Requires<[HasAddr64]>; 129 130// Select waits with no constant offset. 131multiclass WaitPatNoOffset<ValueType ty, Intrinsic kind, 132 string inst> { 133 def : Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), 134 (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, I64:$timeout)>, 135 Requires<[HasAddr32]>; 136 def : Pat<(i32 (kind I64:$addr, ty:$exp, I64:$timeout)), 137 (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, I64:$timeout)>, 138 Requires<[HasAddr64]>; 139} 140defm : WaitPatNoOffset<i32, int_wasm_memory_atomic_wait32, 141 "MEMORY_ATOMIC_WAIT32">; 142defm : WaitPatNoOffset<i64, int_wasm_memory_atomic_wait64, 143 "MEMORY_ATOMIC_WAIT64">; 144defm : WaitPatNoOffset<i32, int_wasm_memory_atomic_wait32, 145 "MEMORY_ATOMIC_WAIT32">; 146defm : WaitPatNoOffset<i64, int_wasm_memory_atomic_wait64, 147 "MEMORY_ATOMIC_WAIT64">; 148 149// Select waits with a constant offset. 150 151// Pattern with address + immediate offset 152multiclass WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, 153 string inst> { 154 def : Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), 155 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, 156 I64:$timeout)>, 157 Requires<[HasAddr32]>; 158 def : Pat<(i32 (kind (operand I64:$addr, imm:$off), ty:$exp, I64:$timeout)), 159 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, 160 I64:$timeout)>, 161 Requires<[HasAddr64]>; 162} 163defm : WaitPatImmOff<i32, int_wasm_memory_atomic_wait32, regPlusImm, 164 "MEMORY_ATOMIC_WAIT32">; 165defm : WaitPatImmOff<i32, int_wasm_memory_atomic_wait32, or_is_add, 166 "MEMORY_ATOMIC_WAIT32">; 167defm : WaitPatImmOff<i64, int_wasm_memory_atomic_wait64, regPlusImm, 168 "MEMORY_ATOMIC_WAIT64">; 169defm : WaitPatImmOff<i64, int_wasm_memory_atomic_wait64, or_is_add, 170 "MEMORY_ATOMIC_WAIT64">; 171 172// Select waits with just a constant offset. 173multiclass WaitPatOffsetOnly<ValueType ty, Intrinsic kind, string inst> { 174 def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 175 (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp, 176 I64:$timeout)>, 177 Requires<[HasAddr32]>; 178 def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 179 (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp, 180 I64:$timeout)>, 181 Requires<[HasAddr64]>; 182} 183defm : WaitPatOffsetOnly<i32, int_wasm_memory_atomic_wait32, 184 "MEMORY_ATOMIC_WAIT32">; 185defm : WaitPatOffsetOnly<i64, int_wasm_memory_atomic_wait64, 186 "MEMORY_ATOMIC_WAIT64">; 187 188multiclass WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, string inst> { 189 def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, 190 I64:$timeout)), 191 (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, 192 I64:$timeout)>, 193 Requires<[HasAddr32]>; 194 def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, 195 I64:$timeout)), 196 (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp, 197 I64:$timeout)>, 198 Requires<[HasAddr64]>; 199} 200defm : WaitPatGlobalAddrOffOnly<i32, int_wasm_memory_atomic_wait32, 201 "MEMORY_ATOMIC_WAIT32">; 202defm : WaitPatGlobalAddrOffOnly<i64, int_wasm_memory_atomic_wait64, 203 "MEMORY_ATOMIC_WAIT64">; 204} // Predicates = [HasAtomics] 205 206//===----------------------------------------------------------------------===// 207// Atomic fences 208//===----------------------------------------------------------------------===// 209 210// A compiler fence instruction that prevents reordering of instructions. 211let Defs = [ARGUMENTS] in { 212let isPseudo = 1, hasSideEffects = 1 in 213defm COMPILER_FENCE : ATOMIC_NRI<(outs), (ins), [], "compiler_fence">; 214let hasSideEffects = 1 in 215defm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins i8imm:$flags), [], "atomic.fence", 216 0x03>; 217} // Defs = [ARGUMENTS] 218 219//===----------------------------------------------------------------------===// 220// Atomic loads 221//===----------------------------------------------------------------------===// 222 223multiclass AtomicLoad<WebAssemblyRegClass rc, string name, int atomic_op> { 224 defm "" : WebAssemblyLoad<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 225 [HasAtomics]>; 226} 227 228defm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>; 229defm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>; 230 231// Select loads with no constant offset. 232let Predicates = [HasAtomics] in { 233defm : LoadPatNoOffset<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 234defm : LoadPatNoOffset<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 235 236// Select loads with a constant offset. 237 238// Pattern with address + immediate offset 239defm : LoadPatImmOff<i32, atomic_load_32, regPlusImm, "ATOMIC_LOAD_I32">; 240defm : LoadPatImmOff<i64, atomic_load_64, regPlusImm, "ATOMIC_LOAD_I64">; 241defm : LoadPatImmOff<i32, atomic_load_32, or_is_add, "ATOMIC_LOAD_I32">; 242defm : LoadPatImmOff<i64, atomic_load_64, or_is_add, "ATOMIC_LOAD_I64">; 243 244// Select loads with just a constant offset. 245defm : LoadPatOffsetOnly<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 246defm : LoadPatOffsetOnly<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 247 248defm : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 249defm : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 250 251} // Predicates = [HasAtomics] 252 253// Extending loads. Note that there are only zero-extending atomic loads, no 254// sign-extending loads. 255defm ATOMIC_LOAD8_U_I32 : AtomicLoad<I32, "i32.atomic.load8_u", 0x12>; 256defm ATOMIC_LOAD16_U_I32 : AtomicLoad<I32, "i32.atomic.load16_u", 0x13>; 257defm ATOMIC_LOAD8_U_I64 : AtomicLoad<I64, "i64.atomic.load8_u", 0x14>; 258defm ATOMIC_LOAD16_U_I64 : AtomicLoad<I64, "i64.atomic.load16_u", 0x15>; 259defm ATOMIC_LOAD32_U_I64 : AtomicLoad<I64, "i64.atomic.load32_u", 0x16>; 260 261// Fragments for extending loads. These are different from regular loads because 262// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 263// therefore don't have the extension type field. So instead of matching that, 264// we match the patterns that the type legalizer expands them to. 265 266// We directly match zext patterns and select the zext atomic loads. 267// i32 (zext (i8 (atomic_load_8))) gets legalized to 268// i32 (and (i32 (atomic_load_8)), 255) 269// These can be selected to a single zero-extending atomic load instruction. 270def zext_aload_8_32 : 271 PatFrag<(ops node:$addr), (and (i32 (atomic_load_8 node:$addr)), 255)>; 272def zext_aload_16_32 : 273 PatFrag<(ops node:$addr), (and (i32 (atomic_load_16 node:$addr)), 65535)>; 274// Unlike regular loads, extension to i64 is handled differently than i32. 275// i64 (zext (i8 (atomic_load_8))) gets legalized to 276// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 277def zext_aload_8_64 : 278 PatFrag<(ops node:$addr), 279 (and (i64 (anyext (i32 (atomic_load_8 node:$addr)))), 255)>; 280def zext_aload_16_64 : 281 PatFrag<(ops node:$addr), 282 (and (i64 (anyext (i32 (atomic_load_16 node:$addr)))), 65535)>; 283def zext_aload_32_64 : 284 PatFrag<(ops node:$addr), 285 (zext (i32 (atomic_load node:$addr)))>; 286 287// We don't have single sext atomic load instructions. So for sext loads, we 288// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 289// results) and select a zext load; the next instruction will be sext_inreg 290// which is selected by itself. 291def sext_aload_8_64 : 292 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 293def sext_aload_16_64 : 294 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 295 296let Predicates = [HasAtomics] in { 297// Select zero-extending loads with no constant offset. 298defm : LoadPatNoOffset<i32, zext_aload_8_32, "ATOMIC_LOAD8_U_I32">; 299defm : LoadPatNoOffset<i32, zext_aload_16_32, "ATOMIC_LOAD16_U_I32">; 300defm : LoadPatNoOffset<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 301defm : LoadPatNoOffset<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 302defm : LoadPatNoOffset<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 303 304// Select sign-extending loads with no constant offset 305defm : LoadPatNoOffset<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 306defm : LoadPatNoOffset<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 307defm : LoadPatNoOffset<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 308defm : LoadPatNoOffset<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 309// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s 310 311// Zero-extending loads with constant offset 312defm : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, "ATOMIC_LOAD8_U_I32">; 313defm : LoadPatImmOff<i32, zext_aload_16_32, regPlusImm, "ATOMIC_LOAD16_U_I32">; 314defm : LoadPatImmOff<i32, zext_aload_8_32, or_is_add, "ATOMIC_LOAD8_U_I32">; 315defm : LoadPatImmOff<i32, zext_aload_16_32, or_is_add, "ATOMIC_LOAD16_U_I32">; 316defm : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, "ATOMIC_LOAD8_U_I64">; 317defm : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, "ATOMIC_LOAD16_U_I64">; 318defm : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, "ATOMIC_LOAD32_U_I64">; 319defm : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, "ATOMIC_LOAD8_U_I64">; 320defm : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, "ATOMIC_LOAD16_U_I64">; 321defm : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, "ATOMIC_LOAD32_U_I64">; 322 323// Sign-extending loads with constant offset 324defm : LoadPatImmOff<i32, atomic_load_8, regPlusImm, "ATOMIC_LOAD8_U_I32">; 325defm : LoadPatImmOff<i32, atomic_load_16, regPlusImm, "ATOMIC_LOAD16_U_I32">; 326defm : LoadPatImmOff<i32, atomic_load_8, or_is_add, "ATOMIC_LOAD8_U_I32">; 327defm : LoadPatImmOff<i32, atomic_load_16, or_is_add, "ATOMIC_LOAD16_U_I32">; 328defm : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, "ATOMIC_LOAD8_U_I64">; 329defm : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, "ATOMIC_LOAD16_U_I64">; 330defm : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, "ATOMIC_LOAD8_U_I64">; 331defm : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, "ATOMIC_LOAD16_U_I64">; 332// No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 333 334// Extending loads with just a constant offset 335defm : LoadPatOffsetOnly<i32, zext_aload_8_32, "ATOMIC_LOAD8_U_I32">; 336defm : LoadPatOffsetOnly<i32, zext_aload_16_32, "ATOMIC_LOAD16_U_I32">; 337defm : LoadPatOffsetOnly<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 338defm : LoadPatOffsetOnly<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 339defm : LoadPatOffsetOnly<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 340defm : LoadPatOffsetOnly<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 341defm : LoadPatOffsetOnly<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 342defm : LoadPatOffsetOnly<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 343defm : LoadPatOffsetOnly<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 344 345defm : LoadPatGlobalAddrOffOnly<i32, zext_aload_8_32, "ATOMIC_LOAD8_U_I32">; 346defm : LoadPatGlobalAddrOffOnly<i32, zext_aload_16_32, "ATOMIC_LOAD16_U_I32">; 347defm : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 348defm : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 349defm : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 350defm : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 351defm : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 352defm : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 353defm : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 354 355} // Predicates = [HasAtomics] 356 357//===----------------------------------------------------------------------===// 358// Atomic stores 359//===----------------------------------------------------------------------===// 360 361multiclass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> { 362 defm "" : WebAssemblyStore<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 363 [HasAtomics]>; 364} 365 366defm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>; 367defm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>; 368 369// We need an 'atomic' version of store patterns because store and atomic_store 370// nodes have different operand orders: 371// store: (store $val, $ptr) 372// atomic_store: (store $ptr, $val) 373 374let Predicates = [HasAtomics] in { 375 376// Select stores with no constant offset. 377multiclass AStorePatNoOffset<ValueType ty, PatFrag kind, string inst> { 378 def : Pat<(kind I32:$addr, ty:$val), 379 (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>, 380 Requires<[HasAddr32]>; 381 def : Pat<(kind I64:$addr, ty:$val), 382 (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>, 383 Requires<[HasAddr64]>; 384} 385defm : AStorePatNoOffset<i32, atomic_store_32, "ATOMIC_STORE_I32">; 386defm : AStorePatNoOffset<i64, atomic_store_64, "ATOMIC_STORE_I64">; 387 388// Select stores with a constant offset. 389 390// Pattern with address + immediate offset 391multiclass AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 392 string inst> { 393 def : Pat<(kind (operand I32:$addr, imm:$off), ty:$val), 394 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, 395 Requires<[HasAddr32]>; 396 def : Pat<(kind (operand I64:$addr, imm:$off), ty:$val), 397 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, 398 Requires<[HasAddr64]>; 399} 400defm : AStorePatImmOff<i32, atomic_store_32, regPlusImm, "ATOMIC_STORE_I32">; 401defm : AStorePatImmOff<i64, atomic_store_64, regPlusImm, "ATOMIC_STORE_I64">; 402 403// Select stores with just a constant offset. 404multiclass AStorePatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 405 def : Pat<(kind imm:$off, ty:$val), 406 (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, 407 Requires<[HasAddr32]>; 408 def : Pat<(kind imm:$off, ty:$val), 409 (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, 410 Requires<[HasAddr64]>; 411} 412defm : AStorePatOffsetOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">; 413defm : AStorePatOffsetOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">; 414 415multiclass AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> { 416 def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 417 (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, 418 Requires<[HasAddr32]>; 419 def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 420 (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, 421 Requires<[HasAddr64]>; 422} 423defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">; 424defm : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">; 425 426} // Predicates = [HasAtomics] 427 428// Truncating stores. 429defm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>; 430defm ATOMIC_STORE16_I32 : AtomicStore<I32, "i32.atomic.store16", 0x1a>; 431defm ATOMIC_STORE8_I64 : AtomicStore<I64, "i64.atomic.store8", 0x1b>; 432defm ATOMIC_STORE16_I64 : AtomicStore<I64, "i64.atomic.store16", 0x1c>; 433defm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>; 434 435// Fragments for truncating stores. 436 437// We don't have single truncating atomic store instructions. For 32-bit 438// instructions, we just need to match bare atomic stores. On the other hand, 439// truncating stores from i64 values are once truncated to i32 first. 440class trunc_astore_64<PatFrag kind> : 441 PatFrag<(ops node:$addr, node:$val), 442 (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 443def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 444def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 445def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 446 447let Predicates = [HasAtomics] in { 448 449// Truncating stores with no constant offset 450defm : AStorePatNoOffset<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 451defm : AStorePatNoOffset<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 452defm : AStorePatNoOffset<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 453defm : AStorePatNoOffset<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 454defm : AStorePatNoOffset<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 455 456// Truncating stores with a constant offset 457defm : AStorePatImmOff<i32, atomic_store_8, regPlusImm, "ATOMIC_STORE8_I32">; 458defm : AStorePatImmOff<i32, atomic_store_16, regPlusImm, "ATOMIC_STORE16_I32">; 459defm : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, "ATOMIC_STORE8_I64">; 460defm : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, 461 "ATOMIC_STORE16_I64">; 462defm : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, 463 "ATOMIC_STORE32_I64">; 464defm : AStorePatImmOff<i32, atomic_store_8, or_is_add, "ATOMIC_STORE8_I32">; 465defm : AStorePatImmOff<i32, atomic_store_16, or_is_add, "ATOMIC_STORE16_I32">; 466defm : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, "ATOMIC_STORE8_I64">; 467defm : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, 468 "ATOMIC_STORE16_I64">; 469defm : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, 470 "ATOMIC_STORE32_I64">; 471 472// Truncating stores with just a constant offset 473defm : AStorePatOffsetOnly<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 474defm : AStorePatOffsetOnly<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 475defm : AStorePatOffsetOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 476defm : AStorePatOffsetOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 477defm : AStorePatOffsetOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 478 479defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 480defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 481defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 482defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 483defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 484 485} // Predicates = [HasAtomics] 486 487//===----------------------------------------------------------------------===// 488// Atomic binary read-modify-writes 489//===----------------------------------------------------------------------===// 490 491multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name, 492 int atomic_op> { 493 defm "_A32" : 494 ATOMIC_I<(outs rc:$dst), 495 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 496 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 497 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 498 !strconcat(name, "\t${off}${p2align}"), atomic_op, "false">; 499 defm "_A64" : 500 ATOMIC_I<(outs rc:$dst), 501 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$val), 502 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 503 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 504 !strconcat(name, "\t${off}${p2align}"), atomic_op, "true">; 505} 506 507defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0x1e>; 508defm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0x1f>; 509defm ATOMIC_RMW8_U_ADD_I32 : 510 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0x20>; 511defm ATOMIC_RMW16_U_ADD_I32 : 512 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0x21>; 513defm ATOMIC_RMW8_U_ADD_I64 : 514 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0x22>; 515defm ATOMIC_RMW16_U_ADD_I64 : 516 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0x23>; 517defm ATOMIC_RMW32_U_ADD_I64 : 518 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0x24>; 519 520defm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0x25>; 521defm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0x26>; 522defm ATOMIC_RMW8_U_SUB_I32 : 523 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0x27>; 524defm ATOMIC_RMW16_U_SUB_I32 : 525 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0x28>; 526defm ATOMIC_RMW8_U_SUB_I64 : 527 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0x29>; 528defm ATOMIC_RMW16_U_SUB_I64 : 529 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0x2a>; 530defm ATOMIC_RMW32_U_SUB_I64 : 531 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0x2b>; 532 533defm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0x2c>; 534defm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0x2d>; 535defm ATOMIC_RMW8_U_AND_I32 : 536 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0x2e>; 537defm ATOMIC_RMW16_U_AND_I32 : 538 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0x2f>; 539defm ATOMIC_RMW8_U_AND_I64 : 540 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0x30>; 541defm ATOMIC_RMW16_U_AND_I64 : 542 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0x31>; 543defm ATOMIC_RMW32_U_AND_I64 : 544 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0x32>; 545 546defm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0x33>; 547defm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0x34>; 548defm ATOMIC_RMW8_U_OR_I32 : 549 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0x35>; 550defm ATOMIC_RMW16_U_OR_I32 : 551 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0x36>; 552defm ATOMIC_RMW8_U_OR_I64 : 553 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0x37>; 554defm ATOMIC_RMW16_U_OR_I64 : 555 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0x38>; 556defm ATOMIC_RMW32_U_OR_I64 : 557 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0x39>; 558 559defm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0x3a>; 560defm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0x3b>; 561defm ATOMIC_RMW8_U_XOR_I32 : 562 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0x3c>; 563defm ATOMIC_RMW16_U_XOR_I32 : 564 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0x3d>; 565defm ATOMIC_RMW8_U_XOR_I64 : 566 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0x3e>; 567defm ATOMIC_RMW16_U_XOR_I64 : 568 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0x3f>; 569defm ATOMIC_RMW32_U_XOR_I64 : 570 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0x40>; 571 572defm ATOMIC_RMW_XCHG_I32 : 573 WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0x41>; 574defm ATOMIC_RMW_XCHG_I64 : 575 WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0x42>; 576defm ATOMIC_RMW8_U_XCHG_I32 : 577 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0x43>; 578defm ATOMIC_RMW16_U_XCHG_I32 : 579 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0x44>; 580defm ATOMIC_RMW8_U_XCHG_I64 : 581 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0x45>; 582defm ATOMIC_RMW16_U_XCHG_I64 : 583 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0x46>; 584defm ATOMIC_RMW32_U_XCHG_I64 : 585 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0x47>; 586 587// Select binary RMWs with no constant offset. 588multiclass BinRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> { 589 def : Pat<(ty (kind I32:$addr, ty:$val)), 590 (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>, 591 Requires<[HasAddr32]>; 592 def : Pat<(ty (kind I64:$addr, ty:$val)), 593 (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>, 594 Requires<[HasAddr64]>; 595} 596 597// Select binary RMWs with a constant offset. 598 599// Pattern with address + immediate offset 600multiclass BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 601 string inst> { 602 def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), 603 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, 604 Requires<[HasAddr32]>; 605 def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$val)), 606 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, 607 Requires<[HasAddr64]>; 608} 609 610// Select binary RMWs with just a constant offset. 611multiclass BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 612 def : Pat<(ty (kind imm:$off, ty:$val)), 613 (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, 614 Requires<[HasAddr32]>; 615 def : Pat<(ty (kind imm:$off, ty:$val)), 616 (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, 617 Requires<[HasAddr64]>; 618} 619 620multiclass BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> { 621 def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 622 (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, 623 Requires<[HasAddr32]>; 624 def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 625 (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, 626 Requires<[HasAddr64]>; 627} 628 629// Patterns for various addressing modes. 630multiclass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32, 631 string inst_64> { 632 defm : BinRMWPatNoOffset<i32, rmw_32, inst_32>; 633 defm : BinRMWPatNoOffset<i64, rmw_64, inst_64>; 634 635 defm : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 636 defm : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 637 defm : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 638 defm : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 639 640 defm : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>; 641 defm : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>; 642 643 defm : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 644 defm : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 645} 646 647let Predicates = [HasAtomics] in { 648defm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, 649 "ATOMIC_RMW_ADD_I32", "ATOMIC_RMW_ADD_I64">; 650defm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, 651 "ATOMIC_RMW_SUB_I32", "ATOMIC_RMW_SUB_I64">; 652defm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, 653 "ATOMIC_RMW_AND_I32", "ATOMIC_RMW_AND_I64">; 654defm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, 655 "ATOMIC_RMW_OR_I32", "ATOMIC_RMW_OR_I64">; 656defm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, 657 "ATOMIC_RMW_XOR_I32", "ATOMIC_RMW_XOR_I64">; 658defm : BinRMWPattern<atomic_swap_32, atomic_swap_64, 659 "ATOMIC_RMW_XCHG_I32", "ATOMIC_RMW_XCHG_I64">; 660} // Predicates = [HasAtomics] 661 662// Truncating & zero-extending binary RMW patterns. 663// These are combined patterns of truncating store patterns and zero-extending 664// load patterns above. 665class zext_bin_rmw_8_32<PatFrag kind> : 666 PatFrag<(ops node:$addr, node:$val), 667 (and (i32 (kind node:$addr, node:$val)), 255)>; 668class zext_bin_rmw_16_32<PatFrag kind> : 669 PatFrag<(ops node:$addr, node:$val), 670 (and (i32 (kind node:$addr, node:$val)), 65535)>; 671class zext_bin_rmw_8_64<PatFrag kind> : 672 PatFrag<(ops node:$addr, node:$val), 673 (and (i64 (anyext (i32 (kind node:$addr, 674 (i32 (trunc (i64 node:$val))))))), 255)>; 675class zext_bin_rmw_16_64<PatFrag kind> : 676 PatFrag<(ops node:$addr, node:$val), 677 (and (i64 (anyext (i32 (kind node:$addr, 678 (i32 (trunc (i64 node:$val))))))), 65535)>; 679class zext_bin_rmw_32_64<PatFrag kind> : 680 PatFrag<(ops node:$addr, node:$val), 681 (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 682 683// Truncating & sign-extending binary RMW patterns. 684// These are combined patterns of truncating store patterns and sign-extending 685// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 686// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 687// is selected by itself. 688class sext_bin_rmw_8_32<PatFrag kind> : 689 PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 690class sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 691class sext_bin_rmw_8_64<PatFrag kind> : 692 PatFrag<(ops node:$addr, node:$val), 693 (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 694class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 695// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 696 697// Patterns for various addressing modes for truncating-extending binary RMWs. 698multiclass BinRMWTruncExtPattern< 699 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 700 NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 701 // Truncating-extending binary RMWs with no constant offset 702 defm : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 703 defm : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 704 defm : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 705 defm : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 706 defm : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 707 708 defm : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 709 defm : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 710 defm : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 711 defm : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 712 713 // Truncating-extending binary RMWs with a constant offset 714 defm : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 715 defm : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm, 716 inst16_32>; 717 defm : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 718 defm : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm, 719 inst16_64>; 720 defm : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm, 721 inst32_64>; 722 defm : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 723 defm : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 724 defm : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 725 defm : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 726 defm : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 727 728 defm : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 729 defm : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm, 730 inst16_32>; 731 defm : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 732 defm : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm, 733 inst16_64>; 734 defm : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 735 defm : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 736 defm : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 737 defm : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 738 739 // Truncating-extending binary RMWs with just a constant offset 740 defm : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 741 defm : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 742 defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 743 defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 744 defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 745 746 defm : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 747 defm : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 748 defm : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 749 defm : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 750 751 defm : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 752 defm : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 753 defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 754 defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 755 defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 756 757 defm : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 758 defm : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 759 defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 760 defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 761} 762 763let Predicates = [HasAtomics] in { 764defm : BinRMWTruncExtPattern< 765 atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, 766 "ATOMIC_RMW8_U_ADD_I32", "ATOMIC_RMW16_U_ADD_I32", 767 "ATOMIC_RMW8_U_ADD_I64", "ATOMIC_RMW16_U_ADD_I64", "ATOMIC_RMW32_U_ADD_I64">; 768defm : BinRMWTruncExtPattern< 769 atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, 770 "ATOMIC_RMW8_U_SUB_I32", "ATOMIC_RMW16_U_SUB_I32", 771 "ATOMIC_RMW8_U_SUB_I64", "ATOMIC_RMW16_U_SUB_I64", "ATOMIC_RMW32_U_SUB_I64">; 772defm : BinRMWTruncExtPattern< 773 atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, 774 "ATOMIC_RMW8_U_AND_I32", "ATOMIC_RMW16_U_AND_I32", 775 "ATOMIC_RMW8_U_AND_I64", "ATOMIC_RMW16_U_AND_I64", "ATOMIC_RMW32_U_AND_I64">; 776defm : BinRMWTruncExtPattern< 777 atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, 778 "ATOMIC_RMW8_U_OR_I32", "ATOMIC_RMW16_U_OR_I32", 779 "ATOMIC_RMW8_U_OR_I64", "ATOMIC_RMW16_U_OR_I64", "ATOMIC_RMW32_U_OR_I64">; 780defm : BinRMWTruncExtPattern< 781 atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, 782 "ATOMIC_RMW8_U_XOR_I32", "ATOMIC_RMW16_U_XOR_I32", 783 "ATOMIC_RMW8_U_XOR_I64", "ATOMIC_RMW16_U_XOR_I64", "ATOMIC_RMW32_U_XOR_I64">; 784defm : BinRMWTruncExtPattern< 785 atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, 786 "ATOMIC_RMW8_U_XCHG_I32", "ATOMIC_RMW16_U_XCHG_I32", 787 "ATOMIC_RMW8_U_XCHG_I64", "ATOMIC_RMW16_U_XCHG_I64", 788 "ATOMIC_RMW32_U_XCHG_I64">; 789} // Predicates = [HasAtomics] 790 791//===----------------------------------------------------------------------===// 792// Atomic ternary read-modify-writes 793//===----------------------------------------------------------------------===// 794 795// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 796// flag}. When we use the success flag or both values, we can't make use of i64 797// truncate/extend versions of instructions for now, which is suboptimal. 798// Consider adding a pass after instruction selection that optimizes this case 799// if it is frequent. 800 801multiclass WebAssemblyTerRMW<WebAssemblyRegClass rc, string name, 802 int atomic_op> { 803 defm "_A32" : 804 ATOMIC_I<(outs rc:$dst), 805 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 806 rc:$new_), 807 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 808 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 809 !strconcat(name, "\t${off}${p2align}"), atomic_op, "false">; 810 defm "_A64" : 811 ATOMIC_I<(outs rc:$dst), 812 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$exp, 813 rc:$new_), 814 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 815 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 816 !strconcat(name, "\t${off}${p2align}"), atomic_op, "true">; 817} 818 819defm ATOMIC_RMW_CMPXCHG_I32 : 820 WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0x48>; 821defm ATOMIC_RMW_CMPXCHG_I64 : 822 WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0x49>; 823defm ATOMIC_RMW8_U_CMPXCHG_I32 : 824 WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0x4a>; 825defm ATOMIC_RMW16_U_CMPXCHG_I32 : 826 WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0x4b>; 827defm ATOMIC_RMW8_U_CMPXCHG_I64 : 828 WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0x4c>; 829defm ATOMIC_RMW16_U_CMPXCHG_I64 : 830 WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0x4d>; 831defm ATOMIC_RMW32_U_CMPXCHG_I64 : 832 WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0x4e>; 833 834// Select ternary RMWs with no constant offset. 835multiclass TerRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> { 836 def : Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), 837 (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, ty:$new)>, 838 Requires<[HasAddr32]>; 839 def : Pat<(ty (kind I64:$addr, ty:$exp, ty:$new)), 840 (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, ty:$new)>, 841 Requires<[HasAddr64]>; 842} 843 844// Select ternary RMWs with a constant offset. 845 846// Pattern with address + immediate offset 847multiclass TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 848 string inst> { 849 def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), 850 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>, 851 Requires<[HasAddr32]>; 852 def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$exp, ty:$new)), 853 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, ty:$new)>, 854 Requires<[HasAddr64]>; 855} 856 857// Select ternary RMWs with just a constant offset. 858multiclass TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 859 def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 860 (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp, 861 ty:$new)>; 862 def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 863 (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp, 864 ty:$new)>; 865} 866 867multiclass TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> { 868 def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 869 (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, 870 ty:$new)>, 871 Requires<[HasAddr32]>; 872 def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 873 (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp, 874 ty:$new)>, 875 Requires<[HasAddr64]>; 876} 877 878// Patterns for various addressing modes. 879multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32, 880 string inst_64> { 881 defm : TerRMWPatNoOffset<i32, rmw_32, inst_32>; 882 defm : TerRMWPatNoOffset<i64, rmw_64, inst_64>; 883 884 defm : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 885 defm : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 886 defm : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 887 defm : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 888 889 defm : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; 890 defm : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; 891 892 defm : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 893 defm : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 894} 895 896let Predicates = [HasAtomics] in 897defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, 898 "ATOMIC_RMW_CMPXCHG_I32", "ATOMIC_RMW_CMPXCHG_I64">; 899 900// Truncating & zero-extending ternary RMW patterns. 901// DAG legalization & optimization before instruction selection may introduce 902// additional nodes such as anyext or assertzext depending on operand types. 903class zext_ter_rmw_8_32<PatFrag kind> : 904 PatFrag<(ops node:$addr, node:$exp, node:$new), 905 (and (i32 (kind node:$addr, node:$exp, node:$new)), 255)>; 906class zext_ter_rmw_16_32<PatFrag kind> : 907 PatFrag<(ops node:$addr, node:$exp, node:$new), 908 (and (i32 (kind node:$addr, node:$exp, node:$new)), 65535)>; 909class zext_ter_rmw_8_64<PatFrag kind> : 910 PatFrag<(ops node:$addr, node:$exp, node:$new), 911 (zext (i32 (assertzext (i32 (kind node:$addr, 912 (i32 (trunc (i64 node:$exp))), 913 (i32 (trunc (i64 node:$new))))))))>; 914class zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 915class zext_ter_rmw_32_64<PatFrag kind> : 916 PatFrag<(ops node:$addr, node:$exp, node:$new), 917 (zext (i32 (kind node:$addr, 918 (i32 (trunc (i64 node:$exp))), 919 (i32 (trunc (i64 node:$new))))))>; 920 921// Truncating & sign-extending ternary RMW patterns. 922// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 923// zext RMW; the next instruction will be sext_inreg which is selected by 924// itself. 925class sext_ter_rmw_8_32<PatFrag kind> : 926 PatFrag<(ops node:$addr, node:$exp, node:$new), 927 (kind node:$addr, node:$exp, node:$new)>; 928class sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 929class sext_ter_rmw_8_64<PatFrag kind> : 930 PatFrag<(ops node:$addr, node:$exp, node:$new), 931 (anyext (i32 (assertzext (i32 932 (kind node:$addr, 933 (i32 (trunc (i64 node:$exp))), 934 (i32 (trunc (i64 node:$new))))))))>; 935class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 936// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 937 938// Patterns for various addressing modes for truncating-extending ternary RMWs. 939multiclass TerRMWTruncExtPattern< 940 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 941 string inst8_32, string inst16_32, string inst8_64, string inst16_64, 942 string inst32_64> { 943 // Truncating-extending ternary RMWs with no constant offset 944 defm : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 945 defm : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 946 defm : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 947 defm : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 948 defm : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 949 950 defm : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 951 defm : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 952 defm : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 953 defm : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 954 955 // Truncating-extending ternary RMWs with a constant offset 956 defm : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 957 defm : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, 958 inst16_32>; 959 defm : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 960 defm : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, 961 inst16_64>; 962 defm : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, 963 inst32_64>; 964 defm : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 965 defm : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 966 defm : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 967 defm : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 968 defm : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 969 970 defm : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 971 defm : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, 972 inst16_32>; 973 defm : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 974 defm : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, 975 inst16_64>; 976 defm : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 977 defm : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 978 defm : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 979 defm : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 980 981 // Truncating-extending ternary RMWs with just a constant offset 982 defm : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 983 defm : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 984 defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 985 defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 986 defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 987 988 defm : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 989 defm : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 990 defm : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 991 defm : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 992 993 defm : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 994 defm : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 995 defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 996 defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 997 defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 998 999 defm : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 1000 defm : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 1001 defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 1002 defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 1003} 1004 1005let Predicates = [HasAtomics] in 1006defm : TerRMWTruncExtPattern< 1007 atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, 1008 "ATOMIC_RMW8_U_CMPXCHG_I32", "ATOMIC_RMW16_U_CMPXCHG_I32", 1009 "ATOMIC_RMW8_U_CMPXCHG_I64", "ATOMIC_RMW16_U_CMPXCHG_I64", 1010 "ATOMIC_RMW32_U_CMPXCHG_I64">; 1011