1 //! This module contains the bulk of the interesting code performing the translation between
2 //! WebAssembly and Cranelift IR.
3 //!
4 //! The translation is done in one pass, opcode by opcode. Two main data structures are used during
5 //! code translations: the value stack and the control stack. The value stack mimics the execution
6 //! of the WebAssembly stack machine: each instruction result is pushed onto the stack and
7 //! instruction arguments are popped off the stack. Similarly, when encountering a control flow
8 //! block, it is pushed onto the control stack and popped off when encountering the corresponding
9 //! `End`.
10 //!
11 //! Another data structure, the translation state, records information concerning unreachable code
12 //! status and about if inserting a return at the end of the function is necessary.
13 //!
14 //! Some of the WebAssembly instructions need information about the environment for which they
15 //! are being translated:
16 //!
17 //! - the loads and stores need the memory base address;
18 //! - the `get_global` and `set_global` instructions depend on how the globals are implemented;
19 //! - `memory.size` and `memory.grow` are runtime functions;
20 //! - `call_indirect` has to translate the function index into the address of where this
21 //! is;
22 //!
23 //! That is why `translate_function_body` takes an object having the `WasmRuntime` trait as
24 //! argument.
25 //!
26 //! There is extra complexity associated with translation of 128-bit SIMD instructions.
27 //! Wasm only considers there to be a single 128-bit vector type. But CLIF's type system
28 //! distinguishes different lane configurations, so considers 8X16, 16X8, 32X4 and 64X2 to be
29 //! different types. The result is that, in wasm, it's perfectly OK to take the output of (eg)
30 //! an `add.16x8` and use that as an operand of a `sub.32x4`, without using any cast. But when
31 //! translated into CLIF, that will cause a verifier error due to the apparent type mismatch.
32 //!
33 //! This file works around that problem by liberally inserting `bitcast` instructions in many
34 //! places -- mostly, before the use of vector values, either as arguments to CLIF instructions
35 //! or as block actual parameters. These are no-op casts which nevertheless have different
36 //! input and output types, and are used (mostly) to "convert" 16X8, 32X4 and 64X2-typed vectors
37 //! to the "canonical" type, 8X16. Hence the functions `optionally_bitcast_vector`,
38 //! `bitcast_arguments`, `pop*_with_bitcast`, `canonicalise_then_jump`,
39 //! `canonicalise_then_br{z,nz}`, `is_non_canonical_v128` and `canonicalise_v128_values`.
40 //! Note that the `bitcast*` functions are occasionally used to convert to some type other than
41 //! 8X16, but the `canonicalise*` functions always convert to type 8X16.
42 //!
43 //! Be careful when adding support for new vector instructions. And when adding new jumps, even
44 //! if they are apparently don't have any connection to vectors. Never generate any kind of
45 //! (inter-block) jump directly. Instead use `canonicalise_then_jump` and
46 //! `canonicalise_then_br{z,nz}`.
47 //!
48 //! The use of bitcasts is ugly and inefficient, but currently unavoidable:
49 //!
50 //! * they make the logic in this file fragile: miss out a bitcast for any reason, and there is
51 //! the risk of the system failing in the verifier. At least for debug builds.
52 //!
53 //! * in the new backends, they potentially interfere with pattern matching on CLIF -- the
54 //! patterns need to take into account the presence of bitcast nodes.
55 //!
56 //! * in the new backends, they get translated into machine-level vector-register-copy
57 //! instructions, none of which are actually necessary. We then depend on the register
58 //! allocator to coalesce them all out.
59 //!
60 //! * they increase the total number of CLIF nodes that have to be processed, hence slowing down
61 //! the compilation pipeline. Also, the extra coalescing work generates a slowdown.
62 //!
63 //! A better solution which would avoid all four problems would be to remove the 8X16, 16X8,
64 //! 32X4 and 64X2 types from CLIF and instead have a single V128 type.
65 //!
66 //! For further background see also:
67 //! <https://github.com/bytecodealliance/wasmtime/issues/1147>
68 //! ("Too many raw_bitcasts in SIMD code")
69 //! <https://github.com/bytecodealliance/cranelift/pull/1251>
70 //! ("Add X128 type to represent WebAssembly's V128 type")
71 //! <https://github.com/bytecodealliance/cranelift/pull/1236>
72 //! ("Relax verification to allow I8X16 to act as a default vector type")
73
74 use crate::Reachability;
75 use crate::bounds_checks::{BoundsCheck, bounds_check_and_compute_addr};
76 use crate::func_environ::{Extension, FuncEnvironment};
77 use crate::translate::TargetEnvironment;
78 use crate::translate::environ::StructFieldsVec;
79 use crate::translate::stack::{ControlStackFrame, ElseData};
80 use crate::translate::translation_utils::{
81 block_with_params, blocktype_params_results, f32_translation, f64_translation,
82 };
83 use crate::trap::TranslateTrap;
84 use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
85 use cranelift_codegen::ir::immediates::Offset32;
86 use cranelift_codegen::ir::{
87 self, AtomicRmwOp, ExceptionTag, InstBuilder, JumpTableData, MemFlags, Value, ValueLabel,
88 };
89 use cranelift_codegen::ir::{BlockArg, types::*};
90 use cranelift_codegen::packed_option::ReservedValue;
91 use cranelift_frontend::{FunctionBuilder, Variable};
92 use itertools::Itertools;
93 use smallvec::{SmallVec, ToSmallVec};
94 use std::collections::{HashMap, hash_map};
95 use std::vec::Vec;
96 use wasmparser::{FuncValidator, MemArg, Operator, WasmModuleResources};
97 use wasmtime_environ::{
98 DataIndex, ElemIndex, FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TagIndex, TypeConvert,
99 TypeIndex, WasmHeapType, WasmRefType, WasmResult, WasmValType, wasm_unsupported,
100 };
101
102 /// Given a `Reachability<T>`, unwrap the inner `T` or, when unreachable, set
103 /// `state.reachable = false` and return.
104 ///
105 /// Used in combination with calling `prepare_addr` and `prepare_atomic_addr`
106 /// when we can statically determine that a Wasm access will unconditionally
107 /// trap.
108 macro_rules! unwrap_or_return_unreachable_state {
109 ($environ:ident, $value:expr) => {
110 match $value {
111 Reachability::Reachable(x) => x,
112 Reachability::Unreachable => {
113 $environ.stacks.reachable = false;
114 return Ok(());
115 }
116 }
117 };
118 }
119
120 /// Translates wasm operators into Cranelift IR instructions.
translate_operator( validator: &mut FuncValidator<impl WasmModuleResources>, op: &Operator, operand_types: Option<&[WasmValType]>, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>, ) -> WasmResult<()>121 pub fn translate_operator(
122 validator: &mut FuncValidator<impl WasmModuleResources>,
123 op: &Operator,
124 operand_types: Option<&[WasmValType]>,
125 builder: &mut FunctionBuilder,
126 environ: &mut FuncEnvironment<'_>,
127 ) -> WasmResult<()> {
128 log::trace!("Translating Wasm opcode: {op:?}");
129
130 if !environ.is_reachable() {
131 translate_unreachable_operator(validator, &op, builder, environ)?;
132 return Ok(());
133 }
134
135 // Given that we believe the current block is reachable, the FunctionBuilder ought to agree.
136 debug_assert!(!builder.is_unreachable());
137 let srcloc = builder.srcloc();
138
139 let operand_types = operand_types.unwrap_or_else(|| {
140 panic!("should always have operand types available for valid, reachable ops; op = {op:?}")
141 });
142
143 // This big match treats all Wasm code operators.
144 match op {
145 /********************************** Locals ****************************************
146 * `get_local` and `set_local` are treated as non-SSA variables and will completely
147 * disappear in the Cranelift Code
148 ***********************************************************************************/
149 Operator::LocalGet { local_index } => {
150 let val = builder.use_var(Variable::from_u32(*local_index));
151 environ.stacks.push1(val);
152 let label = ValueLabel::from_u32(*local_index);
153 builder.set_val_label(val, label);
154 }
155 Operator::LocalSet { local_index } => {
156 let mut val = environ.stacks.pop1();
157
158 // Ensure SIMD values are cast to their default Cranelift type, I8x16.
159 let ty = builder.func.dfg.value_type(val);
160 if ty.is_vector() {
161 val = optionally_bitcast_vector(val, I8X16, builder);
162 }
163
164 builder.def_var(Variable::from_u32(*local_index), val);
165 let label = ValueLabel::from_u32(*local_index);
166 builder.set_val_label(val, label);
167 environ.state_slot_local_set(builder, *local_index, val);
168 }
169 Operator::LocalTee { local_index } => {
170 let mut val = environ.stacks.peek1();
171
172 // Ensure SIMD values are cast to their default Cranelift type, I8x16.
173 let ty = builder.func.dfg.value_type(val);
174 if ty.is_vector() {
175 val = optionally_bitcast_vector(val, I8X16, builder);
176 }
177
178 builder.def_var(Variable::from_u32(*local_index), val);
179 let label = ValueLabel::from_u32(*local_index);
180 builder.set_val_label(val, label);
181 environ.state_slot_local_set(builder, *local_index, val);
182 }
183 /********************************** Globals ****************************************
184 * `get_global` and `set_global` are handled by the environment.
185 ***********************************************************************************/
186 Operator::GlobalGet { global_index } => {
187 let global_index = GlobalIndex::from_u32(*global_index);
188 let val = environ.translate_global_get(builder, global_index)?;
189 environ.stacks.push1(val);
190 }
191 Operator::GlobalSet { global_index } => {
192 let global_index = GlobalIndex::from_u32(*global_index);
193 let mut val = environ.stacks.pop1();
194 // Ensure SIMD values are cast to their default Cranelift type, I8x16.
195 if builder.func.dfg.value_type(val).is_vector() {
196 val = optionally_bitcast_vector(val, I8X16, builder);
197 }
198 environ.translate_global_set(builder, global_index, val)?;
199 }
200 /********************************* Stack misc ***************************************
201 * `drop`, `nop`, `unreachable` and `select`.
202 ***********************************************************************************/
203 Operator::Drop => {
204 environ.stacks.pop1();
205 }
206 Operator::Nop => {
207 // We do nothing
208 }
209 Operator::Select
210 | Operator::TypedSelect {
211 // We ignore the explicit type parameter as it is only needed for
212 // validation, which we require to have been performed before
213 // translation.
214 ty: _,
215 } => {
216 let (mut arg1, mut arg2, cond) = environ.stacks.pop3();
217
218 if builder.func.dfg.value_type(arg1).is_vector() {
219 arg1 = optionally_bitcast_vector(arg1, I8X16, builder);
220 }
221 if builder.func.dfg.value_type(arg2).is_vector() {
222 arg2 = optionally_bitcast_vector(arg2, I8X16, builder);
223 }
224
225 let val = builder.ins().select(cond, arg1, arg2);
226
227 // If either of the input types need inclusion in stack maps, then
228 // the result will as well.
229 //
230 // Note that we don't need to check whether the result's type needs
231 // inclusion in stack maps (that would be a conservative over
232 // approximation) because the input types give us more-precise
233 // information than the result type does. For example, the result
234 // does not need inclusion in stack maps in the scenario where both
235 // inputs are `i31ref`s and the result is an `anyref`. Even though
236 // `anyref`s normally do require inclusion in stack maps, in this
237 // particular case, we know that we are dealing with an `anyref`
238 // that doesn't actually require inclusion.
239 if operand_types
240 .iter()
241 .any(|ty| environ.val_ty_needs_stack_map(*ty))
242 {
243 builder.declare_value_needs_stack_map(val);
244 }
245
246 environ.stacks.push1(val);
247 }
248 Operator::Unreachable => {
249 environ.trap(builder, crate::TRAP_UNREACHABLE);
250 environ.stacks.reachable = false;
251 }
252 /***************************** Control flow blocks **********************************
253 * When starting a control flow block, we create a new `Block` that will hold the code
254 * after the block, and we push a frame on the control stack. Depending on the type
255 * of block, we create a new `Block` for the body of the block with an associated
256 * jump instruction.
257 *
258 * The `End` instruction pops the last control frame from the control stack, seals
259 * the destination block (since `br` instructions targeting it only appear inside the
260 * block and have already been translated) and modify the value stack to use the
261 * possible `Block`'s arguments values.
262 ***********************************************************************************/
263 Operator::Block { blockty } => {
264 let (params, results) = blocktype_params_results(validator, *blockty)?;
265 let next = block_with_params(builder, results.clone(), environ)?;
266 environ.stacks.push_block(next, params.len(), results.len());
267 }
268 Operator::Loop { blockty } => {
269 let (params, results) = blocktype_params_results(validator, *blockty)?;
270 let loop_body = block_with_params(builder, params.clone(), environ)?;
271 let next = block_with_params(builder, results.clone(), environ)?;
272 canonicalise_then_jump(builder, loop_body, environ.stacks.peekn(params.len()));
273 environ
274 .stacks
275 .push_loop(loop_body, next, params.len(), results.len());
276
277 // Pop the initial `Block` actuals and replace them with the `Block`'s
278 // params since control flow joins at the top of the loop.
279 environ.stacks.popn(params.len());
280 environ
281 .stacks
282 .stack
283 .extend_from_slice(builder.block_params(loop_body));
284
285 builder.switch_to_block(loop_body);
286 environ.translate_loop_header(builder)?;
287 }
288 Operator::If { blockty } => {
289 let val = environ.stacks.pop1();
290
291 let next_block = builder.create_block();
292 let (params, results) = blocktype_params_results(validator, *blockty)?;
293 let (destination, else_data) = if params.clone().eq(results.clone()) {
294 // It is possible there is no `else` block, so we will only
295 // allocate a block for it if/when we find the `else`. For now,
296 // we if the condition isn't true, then we jump directly to the
297 // destination block following the whole `if...end`. If we do end
298 // up discovering an `else`, then we will allocate a block for it
299 // and go back and patch the jump.
300 let destination = block_with_params(builder, results.clone(), environ)?;
301 let branch_inst = canonicalise_brif(
302 builder,
303 val,
304 next_block,
305 &[],
306 destination,
307 environ.stacks.peekn(params.len()),
308 );
309 (
310 destination,
311 ElseData::NoElse {
312 branch_inst,
313 placeholder: destination,
314 },
315 )
316 } else {
317 // The `if` type signature is not valid without an `else` block,
318 // so we eagerly allocate the `else` block here.
319 let destination = block_with_params(builder, results.clone(), environ)?;
320 let else_block = block_with_params(builder, params.clone(), environ)?;
321 canonicalise_brif(
322 builder,
323 val,
324 next_block,
325 &[],
326 else_block,
327 environ.stacks.peekn(params.len()),
328 );
329 builder.seal_block(else_block);
330 (destination, ElseData::WithElse { else_block })
331 };
332
333 builder.seal_block(next_block); // Only predecessor is the current block.
334 builder.switch_to_block(next_block);
335
336 // Here we append an argument to a Block targeted by an argumentless jump instruction
337 // But in fact there are two cases:
338 // - either the If does not have a Else clause, in that case ty = EmptyBlock
339 // and we add nothing;
340 // - either the If have an Else clause, in that case the destination of this jump
341 // instruction will be changed later when we translate the Else operator.
342 environ.stacks.push_if(
343 destination,
344 else_data,
345 params.len(),
346 results.len(),
347 *blockty,
348 );
349 }
350 Operator::Else => {
351 let i = environ.stacks.control_stack.len() - 1;
352 let reachable = environ.is_reachable();
353 match environ.stacks.control_stack[i] {
354 ControlStackFrame::If {
355 ref else_data,
356 head_is_reachable,
357 ref mut consequent_ends_reachable,
358 num_return_values,
359 blocktype,
360 destination,
361 ..
362 } => {
363 // We finished the consequent, so record its final
364 // reachability state.
365 debug_assert!(consequent_ends_reachable.is_none());
366 *consequent_ends_reachable = Some(reachable);
367
368 if head_is_reachable {
369 // We have a branch from the head of the `if` to the `else`.
370 environ.stacks.reachable = true;
371
372 // Ensure we have a block for the `else` block (it may have
373 // already been pre-allocated, see `ElseData` for details).
374 let else_block = match *else_data {
375 ElseData::NoElse {
376 branch_inst,
377 placeholder,
378 } => {
379 let (params, _results) =
380 blocktype_params_results(validator, blocktype)?;
381 debug_assert_eq!(params.len(), num_return_values);
382 let else_block =
383 block_with_params(builder, params.clone(), environ)?;
384 canonicalise_then_jump(
385 builder,
386 destination,
387 environ.stacks.peekn(params.len()),
388 );
389 environ.stacks.popn(params.len());
390
391 builder.change_jump_destination(
392 branch_inst,
393 placeholder,
394 else_block,
395 );
396 builder.seal_block(else_block);
397 else_block
398 }
399 ElseData::WithElse { else_block } => {
400 canonicalise_then_jump(
401 builder,
402 destination,
403 environ.stacks.peekn(num_return_values),
404 );
405 environ.stacks.popn(num_return_values);
406 else_block
407 }
408 };
409
410 // You might be expecting that we push the parameters for this
411 // `else` block here, something like this:
412 //
413 // state.pushn(&control_stack_frame.params);
414 //
415 // We don't do that because they are already on the top of the stack
416 // for us: we pushed the parameters twice when we saw the initial
417 // `if` so that we wouldn't have to save the parameters in the
418 // `ControlStackFrame` as another `Vec` allocation.
419
420 builder.switch_to_block(else_block);
421
422 // We don't bother updating the control frame's `ElseData`
423 // to `WithElse` because nothing else will read it.
424 }
425 }
426 _ => unreachable!(),
427 }
428 }
429 Operator::End => {
430 let frame = environ.stacks.control_stack.pop().unwrap();
431 let next_block = frame.following_code();
432 let return_count = frame.num_return_values();
433 let return_args = environ.stacks.peekn_mut(return_count);
434
435 canonicalise_then_jump(builder, next_block, return_args);
436 // You might expect that if we just finished an `if` block that
437 // didn't have a corresponding `else` block, then we would clean
438 // up our duplicate set of parameters that we pushed earlier
439 // right here. However, we don't have to explicitly do that,
440 // since we truncate the stack back to the original height
441 // below.
442
443 builder.switch_to_block(next_block);
444 builder.seal_block(next_block);
445
446 // If it is a loop we also have to seal the body loop block
447 if let ControlStackFrame::Loop { header, .. } = frame {
448 builder.seal_block(header)
449 }
450
451 frame.restore_catch_handlers(&mut environ.stacks.handlers, builder);
452
453 frame.truncate_value_stack_to_original_size(
454 &mut environ.stacks.stack,
455 &mut environ.stacks.stack_shape,
456 );
457 environ
458 .stacks
459 .stack
460 .extend_from_slice(builder.block_params(next_block));
461 }
462 /**************************** Branch instructions *********************************
463 * The branch instructions all have as arguments a target nesting level, which
464 * corresponds to how many control stack frames do we have to pop to get the
465 * destination `Block`.
466 *
467 * Once the destination `Block` is found, we sometimes have to declare a certain depth
468 * of the stack unreachable, because some branch instructions are terminator.
469 *
470 * The `br_table` case is much more complicated because Cranelift's `br_table` instruction
471 * does not support jump arguments like all the other branch instructions. That is why, in
472 * the case where we would use jump arguments for every other branch instruction, we
473 * need to split the critical edges leaving the `br_tables` by creating one `Block` per
474 * table destination; the `br_table` will point to these newly created `Blocks` and these
475 * `Block`s contain only a jump instruction pointing to the final destination, this time with
476 * jump arguments.
477 *
478 * This system is also implemented in Cranelift's SSA construction algorithm, because
479 * `use_var` located in a destination `Block` of a `br_table` might trigger the addition
480 * of jump arguments in each predecessor branch instruction, one of which might be a
481 * `br_table`.
482 ***********************************************************************************/
483 Operator::Br { relative_depth } => {
484 let i = environ.stacks.control_stack.len() - 1 - (*relative_depth as usize);
485 let (return_count, br_destination) = {
486 let frame = &mut environ.stacks.control_stack[i];
487 // We signal that all the code that follows until the next End is unreachable
488 frame.set_branched_to_exit();
489 let return_count = if frame.is_loop() {
490 frame.num_param_values()
491 } else {
492 frame.num_return_values()
493 };
494 (return_count, frame.br_destination())
495 };
496 let destination_args = environ.stacks.peekn_mut(return_count);
497 canonicalise_then_jump(builder, br_destination, destination_args);
498 environ.stacks.popn(return_count);
499 environ.stacks.reachable = false;
500 }
501 Operator::BrIf { relative_depth } => translate_br_if(*relative_depth, builder, environ),
502 Operator::BrTable { targets } => {
503 let default = targets.default();
504 let mut min_depth = default;
505 for depth in targets.targets() {
506 let depth = depth?;
507 if depth < min_depth {
508 min_depth = depth;
509 }
510 }
511 let jump_args_count = {
512 let i = environ.stacks.control_stack.len() - 1 - (min_depth as usize);
513 let min_depth_frame = &environ.stacks.control_stack[i];
514 if min_depth_frame.is_loop() {
515 min_depth_frame.num_param_values()
516 } else {
517 min_depth_frame.num_return_values()
518 }
519 };
520 let val = environ.stacks.pop1();
521 let mut data = Vec::with_capacity(targets.len() as usize);
522 if jump_args_count == 0 {
523 // No jump arguments
524 for depth in targets.targets() {
525 let depth = depth?;
526 let block = {
527 let i = environ.stacks.control_stack.len() - 1 - (depth as usize);
528 let frame = &mut environ.stacks.control_stack[i];
529 frame.set_branched_to_exit();
530 frame.br_destination()
531 };
532 data.push(builder.func.dfg.block_call(block, &[]));
533 }
534 let block = {
535 let i = environ.stacks.control_stack.len() - 1 - (default as usize);
536 let frame = &mut environ.stacks.control_stack[i];
537 frame.set_branched_to_exit();
538 frame.br_destination()
539 };
540 let block = builder.func.dfg.block_call(block, &[]);
541 let jt = builder.create_jump_table(JumpTableData::new(block, &data));
542 builder.ins().br_table(val, jt);
543 } else {
544 // Here we have jump arguments, but Cranelift's br_table doesn't support them
545 // We then proceed to split the edges going out of the br_table
546 let return_count = jump_args_count;
547 let mut dest_block_sequence = vec![];
548 let mut dest_block_map = HashMap::new();
549 for depth in targets.targets() {
550 let depth = depth?;
551 let branch_block = match dest_block_map.entry(depth as usize) {
552 hash_map::Entry::Occupied(entry) => *entry.get(),
553 hash_map::Entry::Vacant(entry) => {
554 let block = builder.create_block();
555 dest_block_sequence.push((depth as usize, block));
556 *entry.insert(block)
557 }
558 };
559 data.push(builder.func.dfg.block_call(branch_block, &[]));
560 }
561 let default_branch_block = match dest_block_map.entry(default as usize) {
562 hash_map::Entry::Occupied(entry) => *entry.get(),
563 hash_map::Entry::Vacant(entry) => {
564 let block = builder.create_block();
565 dest_block_sequence.push((default as usize, block));
566 *entry.insert(block)
567 }
568 };
569 let default_branch_block = builder.func.dfg.block_call(default_branch_block, &[]);
570 let jt = builder.create_jump_table(JumpTableData::new(default_branch_block, &data));
571 builder.ins().br_table(val, jt);
572 for (depth, dest_block) in dest_block_sequence {
573 builder.switch_to_block(dest_block);
574 builder.seal_block(dest_block);
575 let real_dest_block = {
576 let i = environ.stacks.control_stack.len() - 1 - depth;
577 let frame = &mut environ.stacks.control_stack[i];
578 frame.set_branched_to_exit();
579 frame.br_destination()
580 };
581 let destination_args = environ.stacks.peekn_mut(return_count);
582 canonicalise_then_jump(builder, real_dest_block, destination_args);
583 }
584 environ.stacks.popn(return_count);
585 }
586 environ.stacks.reachable = false;
587 }
588 Operator::Return => {
589 let return_count = {
590 let frame = &mut environ.stacks.control_stack[0];
591 frame.num_return_values()
592 };
593 {
594 let mut return_args = environ.stacks.peekn(return_count).to_vec();
595 environ.handle_before_return(&return_args, builder);
596 bitcast_wasm_returns(&mut return_args, builder);
597 builder.ins().return_(&return_args);
598 }
599 environ.stacks.popn(return_count);
600 environ.stacks.reachable = false;
601 }
602 /********************************** Exception handling **********************************/
603 Operator::Catch { .. }
604 | Operator::Rethrow { .. }
605 | Operator::Delegate { .. }
606 | Operator::CatchAll => {
607 return Err(wasm_unsupported!(
608 "legacy exception handling proposal is not supported"
609 ));
610 }
611
612 Operator::TryTable { try_table } => {
613 // First, create a block on the control stack. This also
614 // updates the handler state that is attached to all calls
615 // made within this block.
616 let body = builder.create_block();
617 let (params, results) = blocktype_params_results(validator, try_table.ty)?;
618 let next = block_with_params(builder, results.clone(), environ)?;
619 builder.ins().jump(body, []);
620 builder.seal_block(body);
621
622 // For each catch clause, create a block with the
623 // equivalent of `br` to the target (unboxing the exnref
624 // into stack values or pushing it directly, depending on
625 // the kind of clause).
626 let ckpt = environ.stacks.handlers.take_checkpoint();
627 let mut catch_blocks = vec![];
628 // Process in *reverse* order: see the comment on
629 // [`HandlerState`]. In brief, this allows us to unify the
630 // left-to-right matching semantics of a single
631 // `try_table`'s catch clauses with the inside-out
632 // (deepest scope first) semantics of nested `try_table`s.
633 for catch in try_table.catches.iter().rev() {
634 // This will register the block in `state.handlers`
635 // under the appropriate tag.
636 catch_blocks.push(create_catch_block(builder, catch, environ)?);
637 }
638
639 environ.stacks.push_try_table_block(
640 next,
641 catch_blocks,
642 params.len(),
643 results.len(),
644 ckpt,
645 );
646
647 // Continue codegen into the main body block.
648 builder.switch_to_block(body);
649 }
650
651 Operator::Throw { tag_index } => {
652 let tag_index = TagIndex::from_u32(*tag_index);
653 let arity = environ.tag_param_arity(tag_index);
654 let args = environ.stacks.peekn(arity).to_vec();
655 environ.translate_exn_throw(builder, tag_index, &args)?;
656 environ.stacks.popn(arity);
657 environ.stacks.reachable = false;
658 }
659
660 Operator::ThrowRef => {
661 let exnref = environ.stacks.pop1();
662 environ.translate_exn_throw_ref(builder, exnref)?;
663 environ.stacks.reachable = false;
664 }
665
666 /************************************ Calls ****************************************
667 * The call instructions pop off their arguments from the stack and append their
668 * return values to it. `call_indirect` needs environment support because there is an
669 * argument referring to an index in the external functions table of the module.
670 ************************************************************************************/
671 Operator::Call { function_index } => {
672 let function_index = FuncIndex::from_u32(*function_index);
673 let ty = environ.module.functions[function_index]
674 .signature
675 .unwrap_module_type_index();
676 let sig_ref = environ.get_or_create_interned_sig_ref(builder.func, ty);
677 let num_args = environ.num_params_for_func(function_index);
678
679 // Bitcast any vector arguments to their default type, I8X16, before calling.
680 let mut args = environ.stacks.peekn(num_args).to_vec();
681 bitcast_wasm_params(environ, sig_ref, &mut args, builder);
682
683 let inst_results = environ.translate_call(
684 builder,
685 environ.next_srcloc,
686 function_index,
687 sig_ref,
688 &args,
689 )?;
690
691 debug_assert_eq!(
692 inst_results.len(),
693 builder.func.dfg.signatures[sig_ref].returns.len(),
694 "translate_call results should match the call signature"
695 );
696 environ.stacks.popn(num_args);
697 environ.stacks.pushn(&inst_results);
698 }
699 Operator::CallIndirect {
700 type_index,
701 table_index,
702 } => {
703 // `type_index` is the index of the function's signature and
704 // `table_index` is the index of the table to search the function
705 // in.
706 let type_index = TypeIndex::from_u32(*type_index);
707 let sigref = environ.get_or_create_sig_ref(builder.func, type_index);
708 let num_args = environ.num_params_for_function_type(type_index);
709 let callee = environ.stacks.pop1();
710
711 // Bitcast any vector arguments to their default type, I8X16, before calling.
712 let mut args = environ.stacks.peekn(num_args).to_vec();
713 bitcast_wasm_params(environ, sigref, &mut args, builder);
714
715 let inst_results = environ.translate_call_indirect(
716 builder,
717 environ.next_srcloc,
718 validator.features(),
719 TableIndex::from_u32(*table_index),
720 type_index,
721 sigref,
722 callee,
723 &args,
724 )?;
725 let inst_results = match inst_results {
726 Some(results) => results,
727 None => {
728 environ.stacks.reachable = false;
729 return Ok(());
730 }
731 };
732
733 debug_assert_eq!(
734 inst_results.len(),
735 builder.func.dfg.signatures[sigref].returns.len(),
736 "translate_call_indirect results should match the call signature"
737 );
738 environ.stacks.popn(num_args);
739 environ.stacks.pushn(&inst_results);
740 }
741 /******************************* Tail Calls ******************************************
742 * The tail call instructions pop their arguments from the stack and
743 * then permanently transfer control to their callee. The indirect
744 * version requires environment support (while the direct version can
745 * optionally be hooked but doesn't require it) it interacts with the
746 * VM's runtime state via tables.
747 ************************************************************************************/
748 Operator::ReturnCall { function_index } => {
749 let function_index = FuncIndex::from_u32(*function_index);
750 let ty = environ.module.functions[function_index]
751 .signature
752 .unwrap_module_type_index();
753 let sig_ref = environ.get_or_create_interned_sig_ref(builder.func, ty);
754 let num_args = environ.num_params_for_func(function_index);
755
756 // Bitcast any vector arguments to their default type, I8X16, before calling.
757 let mut args = environ.stacks.peekn(num_args).to_vec();
758 bitcast_wasm_params(environ, sig_ref, &mut args, builder);
759
760 environ.translate_return_call(builder, srcloc, function_index, sig_ref, &args)?;
761
762 environ.stacks.popn(num_args);
763 environ.stacks.reachable = false;
764 }
765 Operator::ReturnCallIndirect {
766 type_index,
767 table_index,
768 } => {
769 // `type_index` is the index of the function's signature and
770 // `table_index` is the index of the table to search the function
771 // in.
772 let type_index = TypeIndex::from_u32(*type_index);
773 let sigref = environ.get_or_create_sig_ref(builder.func, type_index);
774 let num_args = environ.num_params_for_function_type(type_index);
775 let callee = environ.stacks.pop1();
776
777 // Bitcast any vector arguments to their default type, I8X16, before calling.
778 let mut args = environ.stacks.peekn(num_args).to_vec();
779 bitcast_wasm_params(environ, sigref, &mut args, builder);
780
781 environ.translate_return_call_indirect(
782 builder,
783 srcloc,
784 validator.features(),
785 TableIndex::from_u32(*table_index),
786 type_index,
787 sigref,
788 callee,
789 &args,
790 )?;
791
792 environ.stacks.popn(num_args);
793 environ.stacks.reachable = false;
794 }
795 Operator::ReturnCallRef { type_index } => {
796 // Get function signature
797 // `index` is the index of the function's signature and `table_index` is the index of
798 // the table to search the function in.
799 let type_index = TypeIndex::from_u32(*type_index);
800 let sigref = environ.get_or_create_sig_ref(builder.func, type_index);
801 let num_args = environ.num_params_for_function_type(type_index);
802 let callee = environ.stacks.pop1();
803
804 // Bitcast any vector arguments to their default type, I8X16, before calling.
805 let mut args = environ.stacks.peekn(num_args).to_vec();
806 bitcast_wasm_params(environ, sigref, &mut args, builder);
807
808 environ.translate_return_call_ref(builder, srcloc, sigref, callee, &args)?;
809
810 environ.stacks.popn(num_args);
811 environ.stacks.reachable = false;
812 }
813 /******************************* Memory management ***********************************
814 * Memory management is handled by environment. It is usually translated into calls to
815 * special functions.
816 ************************************************************************************/
817 Operator::MemoryGrow { mem } => {
818 // The WebAssembly MVP only supports one linear memory, but we expect the reserved
819 // argument to be a memory index.
820 let mem = MemoryIndex::from_u32(*mem);
821 let _heap = environ.get_or_create_heap(builder.func, mem);
822 let val = environ.stacks.pop1();
823 environ.before_memory_grow(builder, val, mem);
824 let result = environ.translate_memory_grow(builder, mem, val)?;
825 environ.stacks.push1(result);
826 }
827 Operator::MemorySize { mem } => {
828 let mem = MemoryIndex::from_u32(*mem);
829 let _heap = environ.get_or_create_heap(builder.func, mem);
830 let result = environ.translate_memory_size(builder.cursor(), mem)?;
831 environ.stacks.push1(result);
832 }
833 /******************************* Load instructions ***********************************
834 * Wasm specifies an integer alignment flag but we drop it in Cranelift.
835 * The memory base address is provided by the environment.
836 ************************************************************************************/
837 Operator::I32Load8U { memarg } => {
838 unwrap_or_return_unreachable_state!(
839 environ,
840 translate_load(memarg, ir::Opcode::Uload8, I32, builder, environ)?
841 );
842 }
843 Operator::I32Load16U { memarg } => {
844 unwrap_or_return_unreachable_state!(
845 environ,
846 translate_load(memarg, ir::Opcode::Uload16, I32, builder, environ)?
847 );
848 }
849 Operator::I32Load8S { memarg } => {
850 unwrap_or_return_unreachable_state!(
851 environ,
852 translate_load(memarg, ir::Opcode::Sload8, I32, builder, environ)?
853 );
854 }
855 Operator::I32Load16S { memarg } => {
856 unwrap_or_return_unreachable_state!(
857 environ,
858 translate_load(memarg, ir::Opcode::Sload16, I32, builder, environ)?
859 );
860 }
861 Operator::I64Load8U { memarg } => {
862 unwrap_or_return_unreachable_state!(
863 environ,
864 translate_load(memarg, ir::Opcode::Uload8, I64, builder, environ)?
865 );
866 }
867 Operator::I64Load16U { memarg } => {
868 unwrap_or_return_unreachable_state!(
869 environ,
870 translate_load(memarg, ir::Opcode::Uload16, I64, builder, environ)?
871 );
872 }
873 Operator::I64Load8S { memarg } => {
874 unwrap_or_return_unreachable_state!(
875 environ,
876 translate_load(memarg, ir::Opcode::Sload8, I64, builder, environ)?
877 );
878 }
879 Operator::I64Load16S { memarg } => {
880 unwrap_or_return_unreachable_state!(
881 environ,
882 translate_load(memarg, ir::Opcode::Sload16, I64, builder, environ)?
883 );
884 }
885 Operator::I64Load32S { memarg } => {
886 unwrap_or_return_unreachable_state!(
887 environ,
888 translate_load(memarg, ir::Opcode::Sload32, I64, builder, environ)?
889 );
890 }
891 Operator::I64Load32U { memarg } => {
892 unwrap_or_return_unreachable_state!(
893 environ,
894 translate_load(memarg, ir::Opcode::Uload32, I64, builder, environ)?
895 );
896 }
897 Operator::I32Load { memarg } => {
898 unwrap_or_return_unreachable_state!(
899 environ,
900 translate_load(memarg, ir::Opcode::Load, I32, builder, environ)?
901 );
902 }
903 Operator::F32Load { memarg } => {
904 unwrap_or_return_unreachable_state!(
905 environ,
906 translate_load(memarg, ir::Opcode::Load, F32, builder, environ)?
907 );
908 }
909 Operator::I64Load { memarg } => {
910 unwrap_or_return_unreachable_state!(
911 environ,
912 translate_load(memarg, ir::Opcode::Load, I64, builder, environ)?
913 );
914 }
915 Operator::F64Load { memarg } => {
916 unwrap_or_return_unreachable_state!(
917 environ,
918 translate_load(memarg, ir::Opcode::Load, F64, builder, environ)?
919 );
920 }
921 Operator::V128Load { memarg } => {
922 unwrap_or_return_unreachable_state!(
923 environ,
924 translate_load(memarg, ir::Opcode::Load, I8X16, builder, environ)?
925 );
926 }
927 Operator::V128Load8x8S { memarg } => {
928 //TODO(#6829): add before_load() and before_store() hooks for SIMD loads and stores.
929 let (flags, _, base) = unwrap_or_return_unreachable_state!(
930 environ,
931 prepare_addr(memarg, 8, builder, environ)?
932 );
933 let loaded = builder.ins().sload8x8(flags, base, 0);
934 environ.stacks.push1(loaded);
935 }
936 Operator::V128Load8x8U { memarg } => {
937 let (flags, _, base) = unwrap_or_return_unreachable_state!(
938 environ,
939 prepare_addr(memarg, 8, builder, environ)?
940 );
941 let loaded = builder.ins().uload8x8(flags, base, 0);
942 environ.stacks.push1(loaded);
943 }
944 Operator::V128Load16x4S { memarg } => {
945 let (flags, _, base) = unwrap_or_return_unreachable_state!(
946 environ,
947 prepare_addr(memarg, 8, builder, environ)?
948 );
949 let loaded = builder.ins().sload16x4(flags, base, 0);
950 environ.stacks.push1(loaded);
951 }
952 Operator::V128Load16x4U { memarg } => {
953 let (flags, _, base) = unwrap_or_return_unreachable_state!(
954 environ,
955 prepare_addr(memarg, 8, builder, environ)?
956 );
957 let loaded = builder.ins().uload16x4(flags, base, 0);
958 environ.stacks.push1(loaded);
959 }
960 Operator::V128Load32x2S { memarg } => {
961 let (flags, _, base) = unwrap_or_return_unreachable_state!(
962 environ,
963 prepare_addr(memarg, 8, builder, environ)?
964 );
965 let loaded = builder.ins().sload32x2(flags, base, 0);
966 environ.stacks.push1(loaded);
967 }
968 Operator::V128Load32x2U { memarg } => {
969 let (flags, _, base) = unwrap_or_return_unreachable_state!(
970 environ,
971 prepare_addr(memarg, 8, builder, environ)?
972 );
973 let loaded = builder.ins().uload32x2(flags, base, 0);
974 environ.stacks.push1(loaded);
975 }
976 /****************************** Store instructions ***********************************
977 * Wasm specifies an integer alignment flag but we drop it in Cranelift.
978 * The memory base address is provided by the environment.
979 ************************************************************************************/
980 Operator::I32Store { memarg }
981 | Operator::I64Store { memarg }
982 | Operator::F32Store { memarg }
983 | Operator::F64Store { memarg } => {
984 translate_store(memarg, ir::Opcode::Store, builder, environ)?;
985 }
986 Operator::I32Store8 { memarg } | Operator::I64Store8 { memarg } => {
987 translate_store(memarg, ir::Opcode::Istore8, builder, environ)?;
988 }
989 Operator::I32Store16 { memarg } | Operator::I64Store16 { memarg } => {
990 translate_store(memarg, ir::Opcode::Istore16, builder, environ)?;
991 }
992 Operator::I64Store32 { memarg } => {
993 translate_store(memarg, ir::Opcode::Istore32, builder, environ)?;
994 }
995 Operator::V128Store { memarg } => {
996 translate_store(memarg, ir::Opcode::Store, builder, environ)?;
997 }
998 /****************************** Nullary Operators ************************************/
999 Operator::I32Const { value } => {
1000 environ
1001 .stacks
1002 .push1(builder.ins().iconst(I32, i64::from(value.cast_unsigned())));
1003 }
1004 Operator::I64Const { value } => environ.stacks.push1(builder.ins().iconst(I64, *value)),
1005 Operator::F32Const { value } => {
1006 environ
1007 .stacks
1008 .push1(builder.ins().f32const(f32_translation(*value)));
1009 }
1010 Operator::F64Const { value } => {
1011 environ
1012 .stacks
1013 .push1(builder.ins().f64const(f64_translation(*value)));
1014 }
1015 /******************************* Unary Operators *************************************/
1016 Operator::I32Clz | Operator::I64Clz => {
1017 let arg = environ.stacks.pop1();
1018 environ.stacks.push1(builder.ins().clz(arg));
1019 }
1020 Operator::I32Ctz | Operator::I64Ctz => {
1021 let arg = environ.stacks.pop1();
1022 environ.stacks.push1(builder.ins().ctz(arg));
1023 }
1024 Operator::I32Popcnt | Operator::I64Popcnt => {
1025 let arg = environ.stacks.pop1();
1026 environ.stacks.push1(builder.ins().popcnt(arg));
1027 }
1028 Operator::I64ExtendI32S => {
1029 let val = environ.stacks.pop1();
1030 environ.stacks.push1(builder.ins().sextend(I64, val));
1031 }
1032 Operator::I64ExtendI32U => {
1033 let val = environ.stacks.pop1();
1034 environ.stacks.push1(builder.ins().uextend(I64, val));
1035 }
1036 Operator::I32WrapI64 => {
1037 let val = environ.stacks.pop1();
1038 environ.stacks.push1(builder.ins().ireduce(I32, val));
1039 }
1040 Operator::F32Sqrt | Operator::F64Sqrt => {
1041 let arg = environ.stacks.pop1();
1042 environ.stacks.push1(builder.ins().sqrt(arg));
1043 }
1044 Operator::F32Ceil => {
1045 let arg = environ.stacks.pop1();
1046 let result = environ.ceil_f32(builder, arg);
1047 environ.stacks.push1(result);
1048 }
1049 Operator::F64Ceil => {
1050 let arg = environ.stacks.pop1();
1051 let result = environ.ceil_f64(builder, arg);
1052 environ.stacks.push1(result);
1053 }
1054 Operator::F32Floor => {
1055 let arg = environ.stacks.pop1();
1056 let result = environ.floor_f32(builder, arg);
1057 environ.stacks.push1(result);
1058 }
1059 Operator::F64Floor => {
1060 let arg = environ.stacks.pop1();
1061 let result = environ.floor_f64(builder, arg);
1062 environ.stacks.push1(result);
1063 }
1064 Operator::F32Trunc => {
1065 let arg = environ.stacks.pop1();
1066 let result = environ.trunc_f32(builder, arg);
1067 environ.stacks.push1(result);
1068 }
1069 Operator::F64Trunc => {
1070 let arg = environ.stacks.pop1();
1071 let result = environ.trunc_f64(builder, arg);
1072 environ.stacks.push1(result);
1073 }
1074 Operator::F32Nearest => {
1075 let arg = environ.stacks.pop1();
1076 let result = environ.nearest_f32(builder, arg);
1077 environ.stacks.push1(result);
1078 }
1079 Operator::F64Nearest => {
1080 let arg = environ.stacks.pop1();
1081 let result = environ.nearest_f64(builder, arg);
1082 environ.stacks.push1(result);
1083 }
1084 Operator::F32Abs | Operator::F64Abs => {
1085 let val = environ.stacks.pop1();
1086 environ.stacks.push1(builder.ins().fabs(val));
1087 }
1088 Operator::F32Neg | Operator::F64Neg => {
1089 let arg = environ.stacks.pop1();
1090 environ.stacks.push1(builder.ins().fneg(arg));
1091 }
1092 Operator::F64ConvertI64U | Operator::F64ConvertI32U => {
1093 let val = environ.stacks.pop1();
1094 environ.stacks.push1(builder.ins().fcvt_from_uint(F64, val));
1095 }
1096 Operator::F64ConvertI64S | Operator::F64ConvertI32S => {
1097 let val = environ.stacks.pop1();
1098 environ.stacks.push1(builder.ins().fcvt_from_sint(F64, val));
1099 }
1100 Operator::F32ConvertI64S | Operator::F32ConvertI32S => {
1101 let val = environ.stacks.pop1();
1102 environ.stacks.push1(builder.ins().fcvt_from_sint(F32, val));
1103 }
1104 Operator::F32ConvertI64U | Operator::F32ConvertI32U => {
1105 let val = environ.stacks.pop1();
1106 environ.stacks.push1(builder.ins().fcvt_from_uint(F32, val));
1107 }
1108 Operator::F64PromoteF32 => {
1109 let val = environ.stacks.pop1();
1110 environ.stacks.push1(builder.ins().fpromote(F64, val));
1111 }
1112 Operator::F32DemoteF64 => {
1113 let val = environ.stacks.pop1();
1114 environ.stacks.push1(builder.ins().fdemote(F32, val));
1115 }
1116 Operator::I64TruncF64S | Operator::I64TruncF32S => {
1117 let val = environ.stacks.pop1();
1118 let result = environ.translate_fcvt_to_sint(builder, I64, val);
1119 environ.stacks.push1(result);
1120 }
1121 Operator::I32TruncF64S | Operator::I32TruncF32S => {
1122 let val = environ.stacks.pop1();
1123 let result = environ.translate_fcvt_to_sint(builder, I32, val);
1124 environ.stacks.push1(result);
1125 }
1126 Operator::I64TruncF64U | Operator::I64TruncF32U => {
1127 let val = environ.stacks.pop1();
1128 let result = environ.translate_fcvt_to_uint(builder, I64, val);
1129 environ.stacks.push1(result);
1130 }
1131 Operator::I32TruncF64U | Operator::I32TruncF32U => {
1132 let val = environ.stacks.pop1();
1133 let result = environ.translate_fcvt_to_uint(builder, I32, val);
1134 environ.stacks.push1(result);
1135 }
1136 Operator::I64TruncSatF64S | Operator::I64TruncSatF32S => {
1137 let val = environ.stacks.pop1();
1138 environ
1139 .stacks
1140 .push1(builder.ins().fcvt_to_sint_sat(I64, val));
1141 }
1142 Operator::I32TruncSatF64S | Operator::I32TruncSatF32S => {
1143 let val = environ.stacks.pop1();
1144 environ
1145 .stacks
1146 .push1(builder.ins().fcvt_to_sint_sat(I32, val));
1147 }
1148 Operator::I64TruncSatF64U | Operator::I64TruncSatF32U => {
1149 let val = environ.stacks.pop1();
1150 environ
1151 .stacks
1152 .push1(builder.ins().fcvt_to_uint_sat(I64, val));
1153 }
1154 Operator::I32TruncSatF64U | Operator::I32TruncSatF32U => {
1155 let val = environ.stacks.pop1();
1156 environ
1157 .stacks
1158 .push1(builder.ins().fcvt_to_uint_sat(I32, val));
1159 }
1160 Operator::F32ReinterpretI32 => {
1161 let val = environ.stacks.pop1();
1162 environ
1163 .stacks
1164 .push1(builder.ins().bitcast(F32, MemFlags::new(), val));
1165 }
1166 Operator::F64ReinterpretI64 => {
1167 let val = environ.stacks.pop1();
1168 environ
1169 .stacks
1170 .push1(builder.ins().bitcast(F64, MemFlags::new(), val));
1171 }
1172 Operator::I32ReinterpretF32 => {
1173 let val = environ.stacks.pop1();
1174 environ
1175 .stacks
1176 .push1(builder.ins().bitcast(I32, MemFlags::new(), val));
1177 }
1178 Operator::I64ReinterpretF64 => {
1179 let val = environ.stacks.pop1();
1180 environ
1181 .stacks
1182 .push1(builder.ins().bitcast(I64, MemFlags::new(), val));
1183 }
1184 Operator::I32Extend8S => {
1185 let val = environ.stacks.pop1();
1186 environ.stacks.push1(builder.ins().ireduce(I8, val));
1187 let val = environ.stacks.pop1();
1188 environ.stacks.push1(builder.ins().sextend(I32, val));
1189 }
1190 Operator::I32Extend16S => {
1191 let val = environ.stacks.pop1();
1192 environ.stacks.push1(builder.ins().ireduce(I16, val));
1193 let val = environ.stacks.pop1();
1194 environ.stacks.push1(builder.ins().sextend(I32, val));
1195 }
1196 Operator::I64Extend8S => {
1197 let val = environ.stacks.pop1();
1198 environ.stacks.push1(builder.ins().ireduce(I8, val));
1199 let val = environ.stacks.pop1();
1200 environ.stacks.push1(builder.ins().sextend(I64, val));
1201 }
1202 Operator::I64Extend16S => {
1203 let val = environ.stacks.pop1();
1204 environ.stacks.push1(builder.ins().ireduce(I16, val));
1205 let val = environ.stacks.pop1();
1206 environ.stacks.push1(builder.ins().sextend(I64, val));
1207 }
1208 Operator::I64Extend32S => {
1209 let val = environ.stacks.pop1();
1210 environ.stacks.push1(builder.ins().ireduce(I32, val));
1211 let val = environ.stacks.pop1();
1212 environ.stacks.push1(builder.ins().sextend(I64, val));
1213 }
1214 /****************************** Binary Operators ************************************/
1215 Operator::I32Add | Operator::I64Add => {
1216 let (arg1, arg2) = environ.stacks.pop2();
1217 environ.stacks.push1(builder.ins().iadd(arg1, arg2));
1218 }
1219 Operator::I32And | Operator::I64And => {
1220 let (arg1, arg2) = environ.stacks.pop2();
1221 environ.stacks.push1(builder.ins().band(arg1, arg2));
1222 }
1223 Operator::I32Or | Operator::I64Or => {
1224 let (arg1, arg2) = environ.stacks.pop2();
1225 environ.stacks.push1(builder.ins().bor(arg1, arg2));
1226 }
1227 Operator::I32Xor | Operator::I64Xor => {
1228 let (arg1, arg2) = environ.stacks.pop2();
1229 environ.stacks.push1(builder.ins().bxor(arg1, arg2));
1230 }
1231 Operator::I32Shl | Operator::I64Shl => {
1232 let (arg1, arg2) = environ.stacks.pop2();
1233 environ.stacks.push1(builder.ins().ishl(arg1, arg2));
1234 }
1235 Operator::I32ShrS | Operator::I64ShrS => {
1236 let (arg1, arg2) = environ.stacks.pop2();
1237 environ.stacks.push1(builder.ins().sshr(arg1, arg2));
1238 }
1239 Operator::I32ShrU | Operator::I64ShrU => {
1240 let (arg1, arg2) = environ.stacks.pop2();
1241 environ.stacks.push1(builder.ins().ushr(arg1, arg2));
1242 }
1243 Operator::I32Rotl | Operator::I64Rotl => {
1244 let (arg1, arg2) = environ.stacks.pop2();
1245 environ.stacks.push1(builder.ins().rotl(arg1, arg2));
1246 }
1247 Operator::I32Rotr | Operator::I64Rotr => {
1248 let (arg1, arg2) = environ.stacks.pop2();
1249 environ.stacks.push1(builder.ins().rotr(arg1, arg2));
1250 }
1251 Operator::F32Add | Operator::F64Add => {
1252 let (arg1, arg2) = environ.stacks.pop2();
1253 environ.stacks.push1(builder.ins().fadd(arg1, arg2));
1254 }
1255 Operator::I32Sub | Operator::I64Sub => {
1256 let (arg1, arg2) = environ.stacks.pop2();
1257 environ.stacks.push1(builder.ins().isub(arg1, arg2));
1258 }
1259 Operator::F32Sub | Operator::F64Sub => {
1260 let (arg1, arg2) = environ.stacks.pop2();
1261 environ.stacks.push1(builder.ins().fsub(arg1, arg2));
1262 }
1263 Operator::I32Mul | Operator::I64Mul => {
1264 let (arg1, arg2) = environ.stacks.pop2();
1265 environ.stacks.push1(builder.ins().imul(arg1, arg2));
1266 }
1267 Operator::F32Mul | Operator::F64Mul => {
1268 let (arg1, arg2) = environ.stacks.pop2();
1269 environ.stacks.push1(builder.ins().fmul(arg1, arg2));
1270 }
1271 Operator::F32Div | Operator::F64Div => {
1272 let (arg1, arg2) = environ.stacks.pop2();
1273 environ.stacks.push1(builder.ins().fdiv(arg1, arg2));
1274 }
1275 Operator::I32DivS | Operator::I64DivS => {
1276 let (arg1, arg2) = environ.stacks.pop2();
1277 let result = environ.translate_sdiv(builder, arg1, arg2);
1278 environ.stacks.push1(result);
1279 }
1280 Operator::I32DivU | Operator::I64DivU => {
1281 let (arg1, arg2) = environ.stacks.pop2();
1282 let result = environ.translate_udiv(builder, arg1, arg2);
1283 environ.stacks.push1(result);
1284 }
1285 Operator::I32RemS | Operator::I64RemS => {
1286 let (arg1, arg2) = environ.stacks.pop2();
1287 let result = environ.translate_srem(builder, arg1, arg2);
1288 environ.stacks.push1(result);
1289 }
1290 Operator::I32RemU | Operator::I64RemU => {
1291 let (arg1, arg2) = environ.stacks.pop2();
1292 let result = environ.translate_urem(builder, arg1, arg2);
1293 environ.stacks.push1(result);
1294 }
1295 Operator::F32Min | Operator::F64Min => {
1296 let (arg1, arg2) = environ.stacks.pop2();
1297 environ.stacks.push1(builder.ins().fmin(arg1, arg2));
1298 }
1299 Operator::F32Max | Operator::F64Max => {
1300 let (arg1, arg2) = environ.stacks.pop2();
1301 environ.stacks.push1(builder.ins().fmax(arg1, arg2));
1302 }
1303 Operator::F32Copysign | Operator::F64Copysign => {
1304 let (arg1, arg2) = environ.stacks.pop2();
1305 environ.stacks.push1(builder.ins().fcopysign(arg1, arg2));
1306 }
1307 /**************************** Comparison Operators **********************************/
1308 Operator::I32LtS | Operator::I64LtS => {
1309 translate_icmp(IntCC::SignedLessThan, builder, environ)
1310 }
1311 Operator::I32LtU | Operator::I64LtU => {
1312 translate_icmp(IntCC::UnsignedLessThan, builder, environ)
1313 }
1314 Operator::I32LeS | Operator::I64LeS => {
1315 translate_icmp(IntCC::SignedLessThanOrEqual, builder, environ)
1316 }
1317 Operator::I32LeU | Operator::I64LeU => {
1318 translate_icmp(IntCC::UnsignedLessThanOrEqual, builder, environ)
1319 }
1320 Operator::I32GtS | Operator::I64GtS => {
1321 translate_icmp(IntCC::SignedGreaterThan, builder, environ)
1322 }
1323 Operator::I32GtU | Operator::I64GtU => {
1324 translate_icmp(IntCC::UnsignedGreaterThan, builder, environ)
1325 }
1326 Operator::I32GeS | Operator::I64GeS => {
1327 translate_icmp(IntCC::SignedGreaterThanOrEqual, builder, environ)
1328 }
1329 Operator::I32GeU | Operator::I64GeU => {
1330 translate_icmp(IntCC::UnsignedGreaterThanOrEqual, builder, environ)
1331 }
1332 Operator::I32Eqz | Operator::I64Eqz => {
1333 let arg = environ.stacks.pop1();
1334 let val = builder.ins().icmp_imm(IntCC::Equal, arg, 0);
1335 environ.stacks.push1(builder.ins().uextend(I32, val));
1336 }
1337 Operator::I32Eq | Operator::I64Eq => translate_icmp(IntCC::Equal, builder, environ),
1338 Operator::F32Eq | Operator::F64Eq => translate_fcmp(FloatCC::Equal, builder, environ),
1339 Operator::I32Ne | Operator::I64Ne => translate_icmp(IntCC::NotEqual, builder, environ),
1340 Operator::F32Ne | Operator::F64Ne => translate_fcmp(FloatCC::NotEqual, builder, environ),
1341 Operator::F32Gt | Operator::F64Gt => translate_fcmp(FloatCC::GreaterThan, builder, environ),
1342 Operator::F32Ge | Operator::F64Ge => {
1343 translate_fcmp(FloatCC::GreaterThanOrEqual, builder, environ)
1344 }
1345 Operator::F32Lt | Operator::F64Lt => translate_fcmp(FloatCC::LessThan, builder, environ),
1346 Operator::F32Le | Operator::F64Le => {
1347 translate_fcmp(FloatCC::LessThanOrEqual, builder, environ)
1348 }
1349 Operator::RefNull { hty } => {
1350 let hty = environ.convert_heap_type(*hty)?;
1351 let result = environ.translate_ref_null(builder.cursor(), hty)?;
1352 environ.stacks.push1(result);
1353 }
1354 Operator::RefIsNull => {
1355 let value = environ.stacks.pop1();
1356 let [WasmValType::Ref(ty)] = operand_types else {
1357 unreachable!("validation")
1358 };
1359 let result = environ.translate_ref_is_null(builder.cursor(), value, *ty)?;
1360 environ.stacks.push1(result);
1361 }
1362 Operator::RefFunc { function_index } => {
1363 let index = FuncIndex::from_u32(*function_index);
1364 let result = environ.translate_ref_func(builder.cursor(), index)?;
1365 environ.stacks.push1(result);
1366 }
1367 Operator::MemoryAtomicWait32 { memarg } | Operator::MemoryAtomicWait64 { memarg } => {
1368 // The WebAssembly MVP only supports one linear memory and
1369 // wasmparser will ensure that the memory indices specified are
1370 // zero.
1371 let implied_ty = match op {
1372 Operator::MemoryAtomicWait64 { .. } => I64,
1373 Operator::MemoryAtomicWait32 { .. } => I32,
1374 _ => unreachable!(),
1375 };
1376 let memory_index = MemoryIndex::from_u32(memarg.memory);
1377 let heap = environ.get_or_create_heap(builder.func, memory_index);
1378 let timeout = environ.stacks.pop1(); // 64 (fixed)
1379 let expected = environ.stacks.pop1(); // 32 or 64 (per the `Ixx` in `IxxAtomicWait`)
1380 assert!(builder.func.dfg.value_type(expected) == implied_ty);
1381 let addr = environ.stacks.pop1();
1382 let effective_addr = if memarg.offset == 0 {
1383 addr
1384 } else {
1385 let index_type = environ.heaps()[heap].index_type();
1386 let offset = builder.ins().iconst(index_type, memarg.offset as i64);
1387 environ.uadd_overflow_trap(builder, addr, offset, ir::TrapCode::HEAP_OUT_OF_BOUNDS)
1388 };
1389 // `fn translate_atomic_wait` can inspect the type of `expected` to figure out what
1390 // code it needs to generate, if it wants.
1391 let res = environ.translate_atomic_wait(
1392 builder,
1393 memory_index,
1394 heap,
1395 effective_addr,
1396 expected,
1397 timeout,
1398 )?;
1399 environ.stacks.push1(res);
1400 }
1401 Operator::MemoryAtomicNotify { memarg } => {
1402 let memory_index = MemoryIndex::from_u32(memarg.memory);
1403 let heap = environ.get_or_create_heap(builder.func, memory_index);
1404 let count = environ.stacks.pop1(); // 32 (fixed)
1405 let addr = environ.stacks.pop1();
1406 let effective_addr = if memarg.offset == 0 {
1407 addr
1408 } else {
1409 let index_type = environ.heaps()[heap].index_type();
1410 let offset = builder.ins().iconst(index_type, memarg.offset as i64);
1411 environ.uadd_overflow_trap(builder, addr, offset, ir::TrapCode::HEAP_OUT_OF_BOUNDS)
1412 };
1413 let res = environ.translate_atomic_notify(
1414 builder,
1415 memory_index,
1416 heap,
1417 effective_addr,
1418 count,
1419 )?;
1420 environ.stacks.push1(res);
1421 }
1422 Operator::I32AtomicLoad { memarg } => {
1423 translate_atomic_load(I32, I32, memarg, builder, environ)?
1424 }
1425 Operator::I64AtomicLoad { memarg } => {
1426 translate_atomic_load(I64, I64, memarg, builder, environ)?
1427 }
1428 Operator::I32AtomicLoad8U { memarg } => {
1429 translate_atomic_load(I32, I8, memarg, builder, environ)?
1430 }
1431 Operator::I32AtomicLoad16U { memarg } => {
1432 translate_atomic_load(I32, I16, memarg, builder, environ)?
1433 }
1434 Operator::I64AtomicLoad8U { memarg } => {
1435 translate_atomic_load(I64, I8, memarg, builder, environ)?
1436 }
1437 Operator::I64AtomicLoad16U { memarg } => {
1438 translate_atomic_load(I64, I16, memarg, builder, environ)?
1439 }
1440 Operator::I64AtomicLoad32U { memarg } => {
1441 translate_atomic_load(I64, I32, memarg, builder, environ)?
1442 }
1443
1444 Operator::I32AtomicStore { memarg } => {
1445 translate_atomic_store(I32, memarg, builder, environ)?
1446 }
1447 Operator::I64AtomicStore { memarg } => {
1448 translate_atomic_store(I64, memarg, builder, environ)?
1449 }
1450 Operator::I32AtomicStore8 { memarg } => {
1451 translate_atomic_store(I8, memarg, builder, environ)?
1452 }
1453 Operator::I32AtomicStore16 { memarg } => {
1454 translate_atomic_store(I16, memarg, builder, environ)?
1455 }
1456 Operator::I64AtomicStore8 { memarg } => {
1457 translate_atomic_store(I8, memarg, builder, environ)?
1458 }
1459 Operator::I64AtomicStore16 { memarg } => {
1460 translate_atomic_store(I16, memarg, builder, environ)?
1461 }
1462 Operator::I64AtomicStore32 { memarg } => {
1463 translate_atomic_store(I32, memarg, builder, environ)?
1464 }
1465
1466 Operator::I32AtomicRmwAdd { memarg } => {
1467 translate_atomic_rmw(I32, I32, AtomicRmwOp::Add, memarg, builder, environ)?
1468 }
1469 Operator::I64AtomicRmwAdd { memarg } => {
1470 translate_atomic_rmw(I64, I64, AtomicRmwOp::Add, memarg, builder, environ)?
1471 }
1472 Operator::I32AtomicRmw8AddU { memarg } => {
1473 translate_atomic_rmw(I32, I8, AtomicRmwOp::Add, memarg, builder, environ)?
1474 }
1475 Operator::I32AtomicRmw16AddU { memarg } => {
1476 translate_atomic_rmw(I32, I16, AtomicRmwOp::Add, memarg, builder, environ)?
1477 }
1478 Operator::I64AtomicRmw8AddU { memarg } => {
1479 translate_atomic_rmw(I64, I8, AtomicRmwOp::Add, memarg, builder, environ)?
1480 }
1481 Operator::I64AtomicRmw16AddU { memarg } => {
1482 translate_atomic_rmw(I64, I16, AtomicRmwOp::Add, memarg, builder, environ)?
1483 }
1484 Operator::I64AtomicRmw32AddU { memarg } => {
1485 translate_atomic_rmw(I64, I32, AtomicRmwOp::Add, memarg, builder, environ)?
1486 }
1487
1488 Operator::I32AtomicRmwSub { memarg } => {
1489 translate_atomic_rmw(I32, I32, AtomicRmwOp::Sub, memarg, builder, environ)?
1490 }
1491 Operator::I64AtomicRmwSub { memarg } => {
1492 translate_atomic_rmw(I64, I64, AtomicRmwOp::Sub, memarg, builder, environ)?
1493 }
1494 Operator::I32AtomicRmw8SubU { memarg } => {
1495 translate_atomic_rmw(I32, I8, AtomicRmwOp::Sub, memarg, builder, environ)?
1496 }
1497 Operator::I32AtomicRmw16SubU { memarg } => {
1498 translate_atomic_rmw(I32, I16, AtomicRmwOp::Sub, memarg, builder, environ)?
1499 }
1500 Operator::I64AtomicRmw8SubU { memarg } => {
1501 translate_atomic_rmw(I64, I8, AtomicRmwOp::Sub, memarg, builder, environ)?
1502 }
1503 Operator::I64AtomicRmw16SubU { memarg } => {
1504 translate_atomic_rmw(I64, I16, AtomicRmwOp::Sub, memarg, builder, environ)?
1505 }
1506 Operator::I64AtomicRmw32SubU { memarg } => {
1507 translate_atomic_rmw(I64, I32, AtomicRmwOp::Sub, memarg, builder, environ)?
1508 }
1509
1510 Operator::I32AtomicRmwAnd { memarg } => {
1511 translate_atomic_rmw(I32, I32, AtomicRmwOp::And, memarg, builder, environ)?
1512 }
1513 Operator::I64AtomicRmwAnd { memarg } => {
1514 translate_atomic_rmw(I64, I64, AtomicRmwOp::And, memarg, builder, environ)?
1515 }
1516 Operator::I32AtomicRmw8AndU { memarg } => {
1517 translate_atomic_rmw(I32, I8, AtomicRmwOp::And, memarg, builder, environ)?
1518 }
1519 Operator::I32AtomicRmw16AndU { memarg } => {
1520 translate_atomic_rmw(I32, I16, AtomicRmwOp::And, memarg, builder, environ)?
1521 }
1522 Operator::I64AtomicRmw8AndU { memarg } => {
1523 translate_atomic_rmw(I64, I8, AtomicRmwOp::And, memarg, builder, environ)?
1524 }
1525 Operator::I64AtomicRmw16AndU { memarg } => {
1526 translate_atomic_rmw(I64, I16, AtomicRmwOp::And, memarg, builder, environ)?
1527 }
1528 Operator::I64AtomicRmw32AndU { memarg } => {
1529 translate_atomic_rmw(I64, I32, AtomicRmwOp::And, memarg, builder, environ)?
1530 }
1531
1532 Operator::I32AtomicRmwOr { memarg } => {
1533 translate_atomic_rmw(I32, I32, AtomicRmwOp::Or, memarg, builder, environ)?
1534 }
1535 Operator::I64AtomicRmwOr { memarg } => {
1536 translate_atomic_rmw(I64, I64, AtomicRmwOp::Or, memarg, builder, environ)?
1537 }
1538 Operator::I32AtomicRmw8OrU { memarg } => {
1539 translate_atomic_rmw(I32, I8, AtomicRmwOp::Or, memarg, builder, environ)?
1540 }
1541 Operator::I32AtomicRmw16OrU { memarg } => {
1542 translate_atomic_rmw(I32, I16, AtomicRmwOp::Or, memarg, builder, environ)?
1543 }
1544 Operator::I64AtomicRmw8OrU { memarg } => {
1545 translate_atomic_rmw(I64, I8, AtomicRmwOp::Or, memarg, builder, environ)?
1546 }
1547 Operator::I64AtomicRmw16OrU { memarg } => {
1548 translate_atomic_rmw(I64, I16, AtomicRmwOp::Or, memarg, builder, environ)?
1549 }
1550 Operator::I64AtomicRmw32OrU { memarg } => {
1551 translate_atomic_rmw(I64, I32, AtomicRmwOp::Or, memarg, builder, environ)?
1552 }
1553
1554 Operator::I32AtomicRmwXor { memarg } => {
1555 translate_atomic_rmw(I32, I32, AtomicRmwOp::Xor, memarg, builder, environ)?
1556 }
1557 Operator::I64AtomicRmwXor { memarg } => {
1558 translate_atomic_rmw(I64, I64, AtomicRmwOp::Xor, memarg, builder, environ)?
1559 }
1560 Operator::I32AtomicRmw8XorU { memarg } => {
1561 translate_atomic_rmw(I32, I8, AtomicRmwOp::Xor, memarg, builder, environ)?
1562 }
1563 Operator::I32AtomicRmw16XorU { memarg } => {
1564 translate_atomic_rmw(I32, I16, AtomicRmwOp::Xor, memarg, builder, environ)?
1565 }
1566 Operator::I64AtomicRmw8XorU { memarg } => {
1567 translate_atomic_rmw(I64, I8, AtomicRmwOp::Xor, memarg, builder, environ)?
1568 }
1569 Operator::I64AtomicRmw16XorU { memarg } => {
1570 translate_atomic_rmw(I64, I16, AtomicRmwOp::Xor, memarg, builder, environ)?
1571 }
1572 Operator::I64AtomicRmw32XorU { memarg } => {
1573 translate_atomic_rmw(I64, I32, AtomicRmwOp::Xor, memarg, builder, environ)?
1574 }
1575
1576 Operator::I32AtomicRmwXchg { memarg } => {
1577 translate_atomic_rmw(I32, I32, AtomicRmwOp::Xchg, memarg, builder, environ)?
1578 }
1579 Operator::I64AtomicRmwXchg { memarg } => {
1580 translate_atomic_rmw(I64, I64, AtomicRmwOp::Xchg, memarg, builder, environ)?
1581 }
1582 Operator::I32AtomicRmw8XchgU { memarg } => {
1583 translate_atomic_rmw(I32, I8, AtomicRmwOp::Xchg, memarg, builder, environ)?
1584 }
1585 Operator::I32AtomicRmw16XchgU { memarg } => {
1586 translate_atomic_rmw(I32, I16, AtomicRmwOp::Xchg, memarg, builder, environ)?
1587 }
1588 Operator::I64AtomicRmw8XchgU { memarg } => {
1589 translate_atomic_rmw(I64, I8, AtomicRmwOp::Xchg, memarg, builder, environ)?
1590 }
1591 Operator::I64AtomicRmw16XchgU { memarg } => {
1592 translate_atomic_rmw(I64, I16, AtomicRmwOp::Xchg, memarg, builder, environ)?
1593 }
1594 Operator::I64AtomicRmw32XchgU { memarg } => {
1595 translate_atomic_rmw(I64, I32, AtomicRmwOp::Xchg, memarg, builder, environ)?
1596 }
1597
1598 Operator::I32AtomicRmwCmpxchg { memarg } => {
1599 translate_atomic_cas(I32, I32, memarg, builder, environ)?
1600 }
1601 Operator::I64AtomicRmwCmpxchg { memarg } => {
1602 translate_atomic_cas(I64, I64, memarg, builder, environ)?
1603 }
1604 Operator::I32AtomicRmw8CmpxchgU { memarg } => {
1605 translate_atomic_cas(I32, I8, memarg, builder, environ)?
1606 }
1607 Operator::I32AtomicRmw16CmpxchgU { memarg } => {
1608 translate_atomic_cas(I32, I16, memarg, builder, environ)?
1609 }
1610 Operator::I64AtomicRmw8CmpxchgU { memarg } => {
1611 translate_atomic_cas(I64, I8, memarg, builder, environ)?
1612 }
1613 Operator::I64AtomicRmw16CmpxchgU { memarg } => {
1614 translate_atomic_cas(I64, I16, memarg, builder, environ)?
1615 }
1616 Operator::I64AtomicRmw32CmpxchgU { memarg } => {
1617 translate_atomic_cas(I64, I32, memarg, builder, environ)?
1618 }
1619
1620 Operator::AtomicFence { .. } => {
1621 builder.ins().fence();
1622 }
1623 Operator::MemoryCopy { src_mem, dst_mem } => {
1624 let src_index = MemoryIndex::from_u32(*src_mem);
1625 let _src_heap = environ.get_or_create_heap(builder.func, src_index);
1626
1627 let dst_index = MemoryIndex::from_u32(*dst_mem);
1628 let _dst_heap = environ.get_or_create_heap(builder.func, dst_index);
1629
1630 let len = environ.stacks.pop1();
1631 let src_pos = environ.stacks.pop1();
1632 let dst_pos = environ.stacks.pop1();
1633 environ.translate_memory_copy(builder, src_index, dst_index, dst_pos, src_pos, len)?;
1634 }
1635 Operator::MemoryFill { mem } => {
1636 let mem = MemoryIndex::from_u32(*mem);
1637 let _heap = environ.get_or_create_heap(builder.func, mem);
1638 let len = environ.stacks.pop1();
1639 let val = environ.stacks.pop1();
1640 let dest = environ.stacks.pop1();
1641 environ.translate_memory_fill(builder, mem, dest, val, len)?;
1642 }
1643 Operator::MemoryInit { data_index, mem } => {
1644 let mem = MemoryIndex::from_u32(*mem);
1645 let _heap = environ.get_or_create_heap(builder.func, mem);
1646 let len = environ.stacks.pop1();
1647 let src = environ.stacks.pop1();
1648 let dest = environ.stacks.pop1();
1649 environ.translate_memory_init(builder, mem, *data_index, dest, src, len)?;
1650 }
1651 Operator::DataDrop { data_index } => {
1652 environ.translate_data_drop(builder.cursor(), *data_index)?;
1653 }
1654 Operator::TableSize { table: index } => {
1655 let result =
1656 environ.translate_table_size(builder.cursor(), TableIndex::from_u32(*index))?;
1657 environ.stacks.push1(result);
1658 }
1659 Operator::TableGrow { table: index } => {
1660 let table_index = TableIndex::from_u32(*index);
1661 let delta = environ.stacks.pop1();
1662 let init_value = environ.stacks.pop1();
1663 let result = environ.translate_table_grow(builder, table_index, delta, init_value)?;
1664 environ.stacks.push1(result);
1665 }
1666 Operator::TableGet { table: index } => {
1667 let table_index = TableIndex::from_u32(*index);
1668 let index = environ.stacks.pop1();
1669 let result = environ.translate_table_get(builder, table_index, index)?;
1670 environ.stacks.push1(result);
1671 }
1672 Operator::TableSet { table: index } => {
1673 let table_index = TableIndex::from_u32(*index);
1674 let value = environ.stacks.pop1();
1675 let index = environ.stacks.pop1();
1676 environ.translate_table_set(builder, table_index, value, index)?;
1677 }
1678 Operator::TableCopy {
1679 dst_table: dst_table_index,
1680 src_table: src_table_index,
1681 } => {
1682 let len = environ.stacks.pop1();
1683 let src = environ.stacks.pop1();
1684 let dest = environ.stacks.pop1();
1685 environ.translate_table_copy(
1686 builder,
1687 TableIndex::from_u32(*dst_table_index),
1688 TableIndex::from_u32(*src_table_index),
1689 dest,
1690 src,
1691 len,
1692 )?;
1693 }
1694 Operator::TableFill { table } => {
1695 let table_index = TableIndex::from_u32(*table);
1696 let len = environ.stacks.pop1();
1697 let val = environ.stacks.pop1();
1698 let dest = environ.stacks.pop1();
1699 environ.translate_table_fill(builder, table_index, dest, val, len)?;
1700 }
1701 Operator::TableInit {
1702 elem_index,
1703 table: table_index,
1704 } => {
1705 let len = environ.stacks.pop1();
1706 let src = environ.stacks.pop1();
1707 let dest = environ.stacks.pop1();
1708 environ.translate_table_init(
1709 builder,
1710 *elem_index,
1711 TableIndex::from_u32(*table_index),
1712 dest,
1713 src,
1714 len,
1715 )?;
1716 }
1717 Operator::ElemDrop { elem_index } => {
1718 environ.translate_elem_drop(builder.cursor(), *elem_index)?;
1719 }
1720 Operator::V128Const { value } => {
1721 let data = value.bytes().to_vec().into();
1722 let handle = builder.func.dfg.constants.insert(data);
1723 let value = builder.ins().vconst(I8X16, handle);
1724 // the v128.const is typed in CLIF as a I8x16 but bitcast to a different type
1725 // before use
1726 environ.stacks.push1(value)
1727 }
1728 Operator::I8x16Splat | Operator::I16x8Splat => {
1729 let reduced = builder
1730 .ins()
1731 .ireduce(type_of(op).lane_type(), environ.stacks.pop1());
1732 let splatted = builder.ins().splat(type_of(op), reduced);
1733 environ.stacks.push1(splatted)
1734 }
1735 Operator::I32x4Splat
1736 | Operator::I64x2Splat
1737 | Operator::F32x4Splat
1738 | Operator::F64x2Splat => {
1739 let splatted = builder.ins().splat(type_of(op), environ.stacks.pop1());
1740 environ.stacks.push1(splatted)
1741 }
1742 Operator::V128Load8Splat { memarg }
1743 | Operator::V128Load16Splat { memarg }
1744 | Operator::V128Load32Splat { memarg }
1745 | Operator::V128Load64Splat { memarg } => {
1746 unwrap_or_return_unreachable_state!(
1747 environ,
1748 translate_load(
1749 memarg,
1750 ir::Opcode::Load,
1751 type_of(op).lane_type(),
1752 builder,
1753 environ,
1754 )?
1755 );
1756 let splatted = builder.ins().splat(type_of(op), environ.stacks.pop1());
1757 environ.stacks.push1(splatted)
1758 }
1759 Operator::V128Load32Zero { memarg } | Operator::V128Load64Zero { memarg } => {
1760 unwrap_or_return_unreachable_state!(
1761 environ,
1762 translate_load(
1763 memarg,
1764 ir::Opcode::Load,
1765 type_of(op).lane_type(),
1766 builder,
1767 environ,
1768 )?
1769 );
1770 let as_vector = builder
1771 .ins()
1772 .scalar_to_vector(type_of(op), environ.stacks.pop1());
1773 environ.stacks.push1(as_vector)
1774 }
1775 Operator::V128Load8Lane { memarg, lane }
1776 | Operator::V128Load16Lane { memarg, lane }
1777 | Operator::V128Load32Lane { memarg, lane }
1778 | Operator::V128Load64Lane { memarg, lane } => {
1779 let vector = pop1_with_bitcast(environ, type_of(op), builder);
1780 unwrap_or_return_unreachable_state!(
1781 environ,
1782 translate_load(
1783 memarg,
1784 ir::Opcode::Load,
1785 type_of(op).lane_type(),
1786 builder,
1787 environ,
1788 )?
1789 );
1790 let replacement = environ.stacks.pop1();
1791 environ
1792 .stacks
1793 .push1(builder.ins().insertlane(vector, replacement, *lane))
1794 }
1795 Operator::V128Store8Lane { memarg, lane }
1796 | Operator::V128Store16Lane { memarg, lane }
1797 | Operator::V128Store32Lane { memarg, lane }
1798 | Operator::V128Store64Lane { memarg, lane } => {
1799 let vector = pop1_with_bitcast(environ, type_of(op), builder);
1800 environ
1801 .stacks
1802 .push1(builder.ins().extractlane(vector, *lane));
1803 translate_store(memarg, ir::Opcode::Store, builder, environ)?;
1804 }
1805 Operator::I8x16ExtractLaneS { lane } | Operator::I16x8ExtractLaneS { lane } => {
1806 let vector = pop1_with_bitcast(environ, type_of(op), builder);
1807 let extracted = builder.ins().extractlane(vector, *lane);
1808 environ.stacks.push1(builder.ins().sextend(I32, extracted))
1809 }
1810 Operator::I8x16ExtractLaneU { lane } | Operator::I16x8ExtractLaneU { lane } => {
1811 let vector = pop1_with_bitcast(environ, type_of(op), builder);
1812 let extracted = builder.ins().extractlane(vector, *lane);
1813 environ.stacks.push1(builder.ins().uextend(I32, extracted));
1814 // On x86, PEXTRB zeroes the upper bits of the destination register of extractlane so
1815 // uextend could be elided; for now, uextend is needed for Cranelift's type checks to
1816 // work.
1817 }
1818 Operator::I32x4ExtractLane { lane }
1819 | Operator::I64x2ExtractLane { lane }
1820 | Operator::F32x4ExtractLane { lane }
1821 | Operator::F64x2ExtractLane { lane } => {
1822 let vector = pop1_with_bitcast(environ, type_of(op), builder);
1823 environ
1824 .stacks
1825 .push1(builder.ins().extractlane(vector, *lane))
1826 }
1827 Operator::I8x16ReplaceLane { lane } | Operator::I16x8ReplaceLane { lane } => {
1828 let (vector, replacement) = environ.stacks.pop2();
1829 let ty = type_of(op);
1830 let reduced = builder.ins().ireduce(ty.lane_type(), replacement);
1831 let vector = optionally_bitcast_vector(vector, ty, builder);
1832 environ
1833 .stacks
1834 .push1(builder.ins().insertlane(vector, reduced, *lane))
1835 }
1836 Operator::I32x4ReplaceLane { lane }
1837 | Operator::I64x2ReplaceLane { lane }
1838 | Operator::F32x4ReplaceLane { lane }
1839 | Operator::F64x2ReplaceLane { lane } => {
1840 let (vector, replacement) = environ.stacks.pop2();
1841 let vector = optionally_bitcast_vector(vector, type_of(op), builder);
1842 environ
1843 .stacks
1844 .push1(builder.ins().insertlane(vector, replacement, *lane))
1845 }
1846 Operator::I8x16Shuffle { lanes, .. } => {
1847 let (a, b) = pop2_with_bitcast(environ, I8X16, builder);
1848 let result = environ.i8x16_shuffle(builder, a, b, lanes);
1849 environ.stacks.push1(result);
1850 // At this point the original types of a and b are lost; users of this value (i.e. this
1851 // WASM-to-CLIF translator) may need to bitcast for type-correctness. This is due
1852 // to WASM using the less specific v128 type for certain operations and more specific
1853 // types (e.g. i8x16) for others.
1854 }
1855 Operator::I8x16Swizzle => {
1856 let (a, b) = pop2_with_bitcast(environ, I8X16, builder);
1857 let result = environ.swizzle(builder, a, b);
1858 environ.stacks.push1(result);
1859 }
1860 Operator::I8x16Add | Operator::I16x8Add | Operator::I32x4Add | Operator::I64x2Add => {
1861 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1862 environ.stacks.push1(builder.ins().iadd(a, b))
1863 }
1864 Operator::I8x16AddSatS | Operator::I16x8AddSatS => {
1865 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1866 environ.stacks.push1(builder.ins().sadd_sat(a, b))
1867 }
1868 Operator::I8x16AddSatU | Operator::I16x8AddSatU => {
1869 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1870 environ.stacks.push1(builder.ins().uadd_sat(a, b))
1871 }
1872 Operator::I8x16Sub | Operator::I16x8Sub | Operator::I32x4Sub | Operator::I64x2Sub => {
1873 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1874 environ.stacks.push1(builder.ins().isub(a, b))
1875 }
1876 Operator::I8x16SubSatS | Operator::I16x8SubSatS => {
1877 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1878 environ.stacks.push1(builder.ins().ssub_sat(a, b))
1879 }
1880 Operator::I8x16SubSatU | Operator::I16x8SubSatU => {
1881 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1882 environ.stacks.push1(builder.ins().usub_sat(a, b))
1883 }
1884 Operator::I8x16MinS | Operator::I16x8MinS | Operator::I32x4MinS => {
1885 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1886 environ.stacks.push1(builder.ins().smin(a, b))
1887 }
1888 Operator::I8x16MinU | Operator::I16x8MinU | Operator::I32x4MinU => {
1889 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1890 environ.stacks.push1(builder.ins().umin(a, b))
1891 }
1892 Operator::I8x16MaxS | Operator::I16x8MaxS | Operator::I32x4MaxS => {
1893 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1894 environ.stacks.push1(builder.ins().smax(a, b))
1895 }
1896 Operator::I8x16MaxU | Operator::I16x8MaxU | Operator::I32x4MaxU => {
1897 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1898 environ.stacks.push1(builder.ins().umax(a, b))
1899 }
1900 Operator::I8x16AvgrU | Operator::I16x8AvgrU => {
1901 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1902 environ.stacks.push1(builder.ins().avg_round(a, b))
1903 }
1904 Operator::I8x16Neg | Operator::I16x8Neg | Operator::I32x4Neg | Operator::I64x2Neg => {
1905 let a = pop1_with_bitcast(environ, type_of(op), builder);
1906 environ.stacks.push1(builder.ins().ineg(a))
1907 }
1908 Operator::I8x16Abs | Operator::I16x8Abs | Operator::I32x4Abs | Operator::I64x2Abs => {
1909 let a = pop1_with_bitcast(environ, type_of(op), builder);
1910 environ.stacks.push1(builder.ins().iabs(a))
1911 }
1912 Operator::I16x8Mul | Operator::I32x4Mul | Operator::I64x2Mul => {
1913 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1914 environ.stacks.push1(builder.ins().imul(a, b))
1915 }
1916 Operator::V128Or => {
1917 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1918 environ.stacks.push1(builder.ins().bor(a, b))
1919 }
1920 Operator::V128Xor => {
1921 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1922 environ.stacks.push1(builder.ins().bxor(a, b))
1923 }
1924 Operator::V128And => {
1925 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1926 environ.stacks.push1(builder.ins().band(a, b))
1927 }
1928 Operator::V128AndNot => {
1929 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
1930 environ.stacks.push1(builder.ins().band_not(a, b))
1931 }
1932 Operator::V128Not => {
1933 let a = environ.stacks.pop1();
1934 environ.stacks.push1(builder.ins().bnot(a));
1935 }
1936 Operator::I8x16Shl | Operator::I16x8Shl | Operator::I32x4Shl | Operator::I64x2Shl => {
1937 let (a, b) = environ.stacks.pop2();
1938 let bitcast_a = optionally_bitcast_vector(a, type_of(op), builder);
1939 // The spec expects to shift with `b mod lanewidth`; This is directly compatible
1940 // with cranelift's instruction.
1941 environ.stacks.push1(builder.ins().ishl(bitcast_a, b))
1942 }
1943 Operator::I8x16ShrU | Operator::I16x8ShrU | Operator::I32x4ShrU | Operator::I64x2ShrU => {
1944 let (a, b) = environ.stacks.pop2();
1945 let bitcast_a = optionally_bitcast_vector(a, type_of(op), builder);
1946 // The spec expects to shift with `b mod lanewidth`; This is directly compatible
1947 // with cranelift's instruction.
1948 environ.stacks.push1(builder.ins().ushr(bitcast_a, b))
1949 }
1950 Operator::I8x16ShrS | Operator::I16x8ShrS | Operator::I32x4ShrS | Operator::I64x2ShrS => {
1951 let (a, b) = environ.stacks.pop2();
1952 let bitcast_a = optionally_bitcast_vector(a, type_of(op), builder);
1953 // The spec expects to shift with `b mod lanewidth`; This is directly compatible
1954 // with cranelift's instruction.
1955 environ.stacks.push1(builder.ins().sshr(bitcast_a, b))
1956 }
1957 Operator::V128Bitselect => {
1958 let (a, b, c) = pop3_with_bitcast(environ, I8X16, builder);
1959 // The CLIF operand ordering is slightly different and the types of all three
1960 // operands must match (hence the bitcast).
1961 environ.stacks.push1(builder.ins().bitselect(c, a, b))
1962 }
1963 Operator::V128AnyTrue => {
1964 let a = pop1_with_bitcast(environ, type_of(op), builder);
1965 let bool_result = builder.ins().vany_true(a);
1966 environ
1967 .stacks
1968 .push1(builder.ins().uextend(I32, bool_result))
1969 }
1970 Operator::I8x16AllTrue
1971 | Operator::I16x8AllTrue
1972 | Operator::I32x4AllTrue
1973 | Operator::I64x2AllTrue => {
1974 let a = pop1_with_bitcast(environ, type_of(op), builder);
1975 let bool_result = builder.ins().vall_true(a);
1976 environ
1977 .stacks
1978 .push1(builder.ins().uextend(I32, bool_result))
1979 }
1980 Operator::I8x16Bitmask
1981 | Operator::I16x8Bitmask
1982 | Operator::I32x4Bitmask
1983 | Operator::I64x2Bitmask => {
1984 let a = pop1_with_bitcast(environ, type_of(op), builder);
1985 environ.stacks.push1(builder.ins().vhigh_bits(I32, a));
1986 }
1987 Operator::I8x16Eq | Operator::I16x8Eq | Operator::I32x4Eq | Operator::I64x2Eq => {
1988 translate_vector_icmp(IntCC::Equal, type_of(op), builder, environ)
1989 }
1990 Operator::I8x16Ne | Operator::I16x8Ne | Operator::I32x4Ne | Operator::I64x2Ne => {
1991 translate_vector_icmp(IntCC::NotEqual, type_of(op), builder, environ)
1992 }
1993 Operator::I8x16GtS | Operator::I16x8GtS | Operator::I32x4GtS | Operator::I64x2GtS => {
1994 translate_vector_icmp(IntCC::SignedGreaterThan, type_of(op), builder, environ)
1995 }
1996 Operator::I8x16LtS | Operator::I16x8LtS | Operator::I32x4LtS | Operator::I64x2LtS => {
1997 translate_vector_icmp(IntCC::SignedLessThan, type_of(op), builder, environ)
1998 }
1999 Operator::I8x16GtU | Operator::I16x8GtU | Operator::I32x4GtU => {
2000 translate_vector_icmp(IntCC::UnsignedGreaterThan, type_of(op), builder, environ)
2001 }
2002 Operator::I8x16LtU | Operator::I16x8LtU | Operator::I32x4LtU => {
2003 translate_vector_icmp(IntCC::UnsignedLessThan, type_of(op), builder, environ)
2004 }
2005 Operator::I8x16GeS | Operator::I16x8GeS | Operator::I32x4GeS | Operator::I64x2GeS => {
2006 translate_vector_icmp(
2007 IntCC::SignedGreaterThanOrEqual,
2008 type_of(op),
2009 builder,
2010 environ,
2011 )
2012 }
2013 Operator::I8x16LeS | Operator::I16x8LeS | Operator::I32x4LeS | Operator::I64x2LeS => {
2014 translate_vector_icmp(IntCC::SignedLessThanOrEqual, type_of(op), builder, environ)
2015 }
2016 Operator::I8x16GeU | Operator::I16x8GeU | Operator::I32x4GeU => translate_vector_icmp(
2017 IntCC::UnsignedGreaterThanOrEqual,
2018 type_of(op),
2019 builder,
2020 environ,
2021 ),
2022 Operator::I8x16LeU | Operator::I16x8LeU | Operator::I32x4LeU => translate_vector_icmp(
2023 IntCC::UnsignedLessThanOrEqual,
2024 type_of(op),
2025 builder,
2026 environ,
2027 ),
2028 Operator::F32x4Eq | Operator::F64x2Eq => {
2029 translate_vector_fcmp(FloatCC::Equal, type_of(op), builder, environ)
2030 }
2031 Operator::F32x4Ne | Operator::F64x2Ne => {
2032 translate_vector_fcmp(FloatCC::NotEqual, type_of(op), builder, environ)
2033 }
2034 Operator::F32x4Lt | Operator::F64x2Lt => {
2035 translate_vector_fcmp(FloatCC::LessThan, type_of(op), builder, environ)
2036 }
2037 Operator::F32x4Gt | Operator::F64x2Gt => {
2038 translate_vector_fcmp(FloatCC::GreaterThan, type_of(op), builder, environ)
2039 }
2040 Operator::F32x4Le | Operator::F64x2Le => {
2041 translate_vector_fcmp(FloatCC::LessThanOrEqual, type_of(op), builder, environ)
2042 }
2043 Operator::F32x4Ge | Operator::F64x2Ge => {
2044 translate_vector_fcmp(FloatCC::GreaterThanOrEqual, type_of(op), builder, environ)
2045 }
2046 Operator::F32x4Add | Operator::F64x2Add => {
2047 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
2048 environ.stacks.push1(builder.ins().fadd(a, b))
2049 }
2050 Operator::F32x4Sub | Operator::F64x2Sub => {
2051 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
2052 environ.stacks.push1(builder.ins().fsub(a, b))
2053 }
2054 Operator::F32x4Mul | Operator::F64x2Mul => {
2055 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
2056 environ.stacks.push1(builder.ins().fmul(a, b))
2057 }
2058 Operator::F32x4Div | Operator::F64x2Div => {
2059 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
2060 environ.stacks.push1(builder.ins().fdiv(a, b))
2061 }
2062 Operator::F32x4Max | Operator::F64x2Max => {
2063 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
2064 environ.stacks.push1(builder.ins().fmax(a, b))
2065 }
2066 Operator::F32x4Min | Operator::F64x2Min => {
2067 let (a, b) = pop2_with_bitcast(environ, type_of(op), builder);
2068 environ.stacks.push1(builder.ins().fmin(a, b))
2069 }
2070 Operator::F32x4PMax | Operator::F64x2PMax => {
2071 // Note the careful ordering here with respect to `fcmp` and
2072 // `bitselect`. This matches the spec definition of:
2073 //
2074 // fpmax(z1, z2) =
2075 // * If z1 is less than z2 then return z2.
2076 // * Else return z1.
2077 let ty = type_of(op);
2078 let (a, b) = pop2_with_bitcast(environ, ty, builder);
2079 let cmp = builder.ins().fcmp(FloatCC::LessThan, a, b);
2080 let cmp = optionally_bitcast_vector(cmp, ty, builder);
2081 environ.stacks.push1(builder.ins().bitselect(cmp, b, a))
2082 }
2083 Operator::F32x4PMin | Operator::F64x2PMin => {
2084 // Note the careful ordering here which is similar to `pmax` above:
2085 //
2086 // fpmin(z1, z2) =
2087 // * If z2 is less than z1 then return z2.
2088 // * Else return z1.
2089 let ty = type_of(op);
2090 let (a, b) = pop2_with_bitcast(environ, ty, builder);
2091 let cmp = builder.ins().fcmp(FloatCC::LessThan, b, a);
2092 let cmp = optionally_bitcast_vector(cmp, ty, builder);
2093 environ.stacks.push1(builder.ins().bitselect(cmp, b, a))
2094 }
2095 Operator::F32x4Sqrt | Operator::F64x2Sqrt => {
2096 let a = pop1_with_bitcast(environ, type_of(op), builder);
2097 environ.stacks.push1(builder.ins().sqrt(a))
2098 }
2099 Operator::F32x4Neg | Operator::F64x2Neg => {
2100 let a = pop1_with_bitcast(environ, type_of(op), builder);
2101 environ.stacks.push1(builder.ins().fneg(a))
2102 }
2103 Operator::F32x4Abs | Operator::F64x2Abs => {
2104 let a = pop1_with_bitcast(environ, type_of(op), builder);
2105 environ.stacks.push1(builder.ins().fabs(a))
2106 }
2107 Operator::F32x4ConvertI32x4S => {
2108 let a = pop1_with_bitcast(environ, I32X4, builder);
2109 environ.stacks.push1(builder.ins().fcvt_from_sint(F32X4, a))
2110 }
2111 Operator::F32x4ConvertI32x4U => {
2112 let a = pop1_with_bitcast(environ, I32X4, builder);
2113 environ.stacks.push1(builder.ins().fcvt_from_uint(F32X4, a))
2114 }
2115 Operator::F64x2ConvertLowI32x4S => {
2116 let a = pop1_with_bitcast(environ, I32X4, builder);
2117 let widened_a = builder.ins().swiden_low(a);
2118 environ
2119 .stacks
2120 .push1(builder.ins().fcvt_from_sint(F64X2, widened_a));
2121 }
2122 Operator::F64x2ConvertLowI32x4U => {
2123 let a = pop1_with_bitcast(environ, I32X4, builder);
2124 let widened_a = builder.ins().uwiden_low(a);
2125 environ
2126 .stacks
2127 .push1(builder.ins().fcvt_from_uint(F64X2, widened_a));
2128 }
2129 Operator::F64x2PromoteLowF32x4 => {
2130 let a = pop1_with_bitcast(environ, F32X4, builder);
2131 environ.stacks.push1(builder.ins().fvpromote_low(a));
2132 }
2133 Operator::F32x4DemoteF64x2Zero => {
2134 let a = pop1_with_bitcast(environ, F64X2, builder);
2135 environ.stacks.push1(builder.ins().fvdemote(a));
2136 }
2137 Operator::I32x4TruncSatF32x4S => {
2138 let a = pop1_with_bitcast(environ, F32X4, builder);
2139 environ
2140 .stacks
2141 .push1(builder.ins().fcvt_to_sint_sat(I32X4, a))
2142 }
2143 Operator::I32x4TruncSatF64x2SZero => {
2144 let a = pop1_with_bitcast(environ, F64X2, builder);
2145 let converted_a = builder.ins().fcvt_to_sint_sat(I64X2, a);
2146 let handle = builder.func.dfg.constants.insert(vec![0u8; 16].into());
2147 let zero = builder.ins().vconst(I64X2, handle);
2148
2149 environ
2150 .stacks
2151 .push1(builder.ins().snarrow(converted_a, zero));
2152 }
2153
2154 // FIXME(#5913): the relaxed instructions here are translated the same
2155 // as the saturating instructions, even when the code generator
2156 // configuration allow for different semantics across hosts. On x86,
2157 // however, it's theoretically possible to have a slightly more optimal
2158 // lowering which accounts for NaN differently, although the lowering is
2159 // still not trivial (e.g. one instruction). At this time the
2160 // more-optimal-but-still-large lowering for x86 is not implemented so
2161 // the relaxed instructions are listed here instead of down below with
2162 // the other relaxed instructions. An x86-specific implementation (or
2163 // perhaps for other backends too) should be added and the codegen for
2164 // the relaxed instruction should conditionally be different.
2165 Operator::I32x4RelaxedTruncF32x4U | Operator::I32x4TruncSatF32x4U => {
2166 let a = pop1_with_bitcast(environ, F32X4, builder);
2167 environ
2168 .stacks
2169 .push1(builder.ins().fcvt_to_uint_sat(I32X4, a))
2170 }
2171 Operator::I32x4RelaxedTruncF64x2UZero | Operator::I32x4TruncSatF64x2UZero => {
2172 let a = pop1_with_bitcast(environ, F64X2, builder);
2173 let zero_constant = builder.func.dfg.constants.insert(vec![0u8; 16].into());
2174 let result = if environ.is_x86() && !environ.isa().has_round() {
2175 // On x86 the vector lowering for `fcvt_to_uint_sat` requires
2176 // SSE4.1 `round` instructions. If SSE4.1 isn't available it
2177 // falls back to a libcall which we don't want in Wasmtime.
2178 // Handle this by falling back to the scalar implementation
2179 // which does not require SSE4.1 instructions.
2180 let lane0 = builder.ins().extractlane(a, 0);
2181 let lane1 = builder.ins().extractlane(a, 1);
2182 let lane0_rounded = builder.ins().fcvt_to_uint_sat(I32, lane0);
2183 let lane1_rounded = builder.ins().fcvt_to_uint_sat(I32, lane1);
2184 let result = builder.ins().vconst(I32X4, zero_constant);
2185 let result = builder.ins().insertlane(result, lane0_rounded, 0);
2186 builder.ins().insertlane(result, lane1_rounded, 1)
2187 } else {
2188 let converted_a = builder.ins().fcvt_to_uint_sat(I64X2, a);
2189 let zero = builder.ins().vconst(I64X2, zero_constant);
2190 builder.ins().uunarrow(converted_a, zero)
2191 };
2192 environ.stacks.push1(result);
2193 }
2194
2195 Operator::I8x16NarrowI16x8S => {
2196 let (a, b) = pop2_with_bitcast(environ, I16X8, builder);
2197 environ.stacks.push1(builder.ins().snarrow(a, b))
2198 }
2199 Operator::I16x8NarrowI32x4S => {
2200 let (a, b) = pop2_with_bitcast(environ, I32X4, builder);
2201 environ.stacks.push1(builder.ins().snarrow(a, b))
2202 }
2203 Operator::I8x16NarrowI16x8U => {
2204 let (a, b) = pop2_with_bitcast(environ, I16X8, builder);
2205 environ.stacks.push1(builder.ins().unarrow(a, b))
2206 }
2207 Operator::I16x8NarrowI32x4U => {
2208 let (a, b) = pop2_with_bitcast(environ, I32X4, builder);
2209 environ.stacks.push1(builder.ins().unarrow(a, b))
2210 }
2211 Operator::I16x8ExtendLowI8x16S => {
2212 let a = pop1_with_bitcast(environ, I8X16, builder);
2213 environ.stacks.push1(builder.ins().swiden_low(a))
2214 }
2215 Operator::I16x8ExtendHighI8x16S => {
2216 let a = pop1_with_bitcast(environ, I8X16, builder);
2217 environ.stacks.push1(builder.ins().swiden_high(a))
2218 }
2219 Operator::I16x8ExtendLowI8x16U => {
2220 let a = pop1_with_bitcast(environ, I8X16, builder);
2221 environ.stacks.push1(builder.ins().uwiden_low(a))
2222 }
2223 Operator::I16x8ExtendHighI8x16U => {
2224 let a = pop1_with_bitcast(environ, I8X16, builder);
2225 environ.stacks.push1(builder.ins().uwiden_high(a))
2226 }
2227 Operator::I32x4ExtendLowI16x8S => {
2228 let a = pop1_with_bitcast(environ, I16X8, builder);
2229 environ.stacks.push1(builder.ins().swiden_low(a))
2230 }
2231 Operator::I32x4ExtendHighI16x8S => {
2232 let a = pop1_with_bitcast(environ, I16X8, builder);
2233 environ.stacks.push1(builder.ins().swiden_high(a))
2234 }
2235 Operator::I32x4ExtendLowI16x8U => {
2236 let a = pop1_with_bitcast(environ, I16X8, builder);
2237 environ.stacks.push1(builder.ins().uwiden_low(a))
2238 }
2239 Operator::I32x4ExtendHighI16x8U => {
2240 let a = pop1_with_bitcast(environ, I16X8, builder);
2241 environ.stacks.push1(builder.ins().uwiden_high(a))
2242 }
2243 Operator::I64x2ExtendLowI32x4S => {
2244 let a = pop1_with_bitcast(environ, I32X4, builder);
2245 environ.stacks.push1(builder.ins().swiden_low(a))
2246 }
2247 Operator::I64x2ExtendHighI32x4S => {
2248 let a = pop1_with_bitcast(environ, I32X4, builder);
2249 environ.stacks.push1(builder.ins().swiden_high(a))
2250 }
2251 Operator::I64x2ExtendLowI32x4U => {
2252 let a = pop1_with_bitcast(environ, I32X4, builder);
2253 environ.stacks.push1(builder.ins().uwiden_low(a))
2254 }
2255 Operator::I64x2ExtendHighI32x4U => {
2256 let a = pop1_with_bitcast(environ, I32X4, builder);
2257 environ.stacks.push1(builder.ins().uwiden_high(a))
2258 }
2259 Operator::I16x8ExtAddPairwiseI8x16S => {
2260 let a = pop1_with_bitcast(environ, I8X16, builder);
2261 let widen_low = builder.ins().swiden_low(a);
2262 let widen_high = builder.ins().swiden_high(a);
2263 environ
2264 .stacks
2265 .push1(builder.ins().iadd_pairwise(widen_low, widen_high));
2266 }
2267 Operator::I32x4ExtAddPairwiseI16x8S => {
2268 let a = pop1_with_bitcast(environ, I16X8, builder);
2269 let widen_low = builder.ins().swiden_low(a);
2270 let widen_high = builder.ins().swiden_high(a);
2271 environ
2272 .stacks
2273 .push1(builder.ins().iadd_pairwise(widen_low, widen_high));
2274 }
2275 Operator::I16x8ExtAddPairwiseI8x16U => {
2276 let a = pop1_with_bitcast(environ, I8X16, builder);
2277 let widen_low = builder.ins().uwiden_low(a);
2278 let widen_high = builder.ins().uwiden_high(a);
2279 environ
2280 .stacks
2281 .push1(builder.ins().iadd_pairwise(widen_low, widen_high));
2282 }
2283 Operator::I32x4ExtAddPairwiseI16x8U => {
2284 let a = pop1_with_bitcast(environ, I16X8, builder);
2285 let widen_low = builder.ins().uwiden_low(a);
2286 let widen_high = builder.ins().uwiden_high(a);
2287 environ
2288 .stacks
2289 .push1(builder.ins().iadd_pairwise(widen_low, widen_high));
2290 }
2291 Operator::F32x4Ceil => {
2292 let arg = pop1_with_bitcast(environ, F32X4, builder);
2293 let result = environ.ceil_f32x4(builder, arg);
2294 environ.stacks.push1(result);
2295 }
2296 Operator::F64x2Ceil => {
2297 let arg = pop1_with_bitcast(environ, F64X2, builder);
2298 let result = environ.ceil_f64x2(builder, arg);
2299 environ.stacks.push1(result);
2300 }
2301 Operator::F32x4Floor => {
2302 let arg = pop1_with_bitcast(environ, F32X4, builder);
2303 let result = environ.floor_f32x4(builder, arg);
2304 environ.stacks.push1(result);
2305 }
2306 Operator::F64x2Floor => {
2307 let arg = pop1_with_bitcast(environ, F64X2, builder);
2308 let result = environ.floor_f64x2(builder, arg);
2309 environ.stacks.push1(result);
2310 }
2311 Operator::F32x4Trunc => {
2312 let arg = pop1_with_bitcast(environ, F32X4, builder);
2313 let result = environ.trunc_f32x4(builder, arg);
2314 environ.stacks.push1(result);
2315 }
2316 Operator::F64x2Trunc => {
2317 let arg = pop1_with_bitcast(environ, F64X2, builder);
2318 let result = environ.trunc_f64x2(builder, arg);
2319 environ.stacks.push1(result);
2320 }
2321 Operator::F32x4Nearest => {
2322 let arg = pop1_with_bitcast(environ, F32X4, builder);
2323 let result = environ.nearest_f32x4(builder, arg);
2324 environ.stacks.push1(result);
2325 }
2326 Operator::F64x2Nearest => {
2327 let arg = pop1_with_bitcast(environ, F64X2, builder);
2328 let result = environ.nearest_f64x2(builder, arg);
2329 environ.stacks.push1(result);
2330 }
2331 Operator::I32x4DotI16x8S => {
2332 let (a, b) = pop2_with_bitcast(environ, I16X8, builder);
2333 let alow = builder.ins().swiden_low(a);
2334 let blow = builder.ins().swiden_low(b);
2335 let low = builder.ins().imul(alow, blow);
2336 let ahigh = builder.ins().swiden_high(a);
2337 let bhigh = builder.ins().swiden_high(b);
2338 let high = builder.ins().imul(ahigh, bhigh);
2339 environ.stacks.push1(builder.ins().iadd_pairwise(low, high));
2340 }
2341 Operator::I8x16Popcnt => {
2342 let arg = pop1_with_bitcast(environ, type_of(op), builder);
2343 environ.stacks.push1(builder.ins().popcnt(arg));
2344 }
2345 Operator::I16x8Q15MulrSatS => {
2346 let (a, b) = pop2_with_bitcast(environ, I16X8, builder);
2347 environ.stacks.push1(builder.ins().sqmul_round_sat(a, b))
2348 }
2349 Operator::I16x8ExtMulLowI8x16S => {
2350 let (a, b) = pop2_with_bitcast(environ, I8X16, builder);
2351 let a_low = builder.ins().swiden_low(a);
2352 let b_low = builder.ins().swiden_low(b);
2353 environ.stacks.push1(builder.ins().imul(a_low, b_low));
2354 }
2355 Operator::I16x8ExtMulHighI8x16S => {
2356 let (a, b) = pop2_with_bitcast(environ, I8X16, builder);
2357 let a_high = builder.ins().swiden_high(a);
2358 let b_high = builder.ins().swiden_high(b);
2359 environ.stacks.push1(builder.ins().imul(a_high, b_high));
2360 }
2361 Operator::I16x8ExtMulLowI8x16U => {
2362 let (a, b) = pop2_with_bitcast(environ, I8X16, builder);
2363 let a_low = builder.ins().uwiden_low(a);
2364 let b_low = builder.ins().uwiden_low(b);
2365 environ.stacks.push1(builder.ins().imul(a_low, b_low));
2366 }
2367 Operator::I16x8ExtMulHighI8x16U => {
2368 let (a, b) = pop2_with_bitcast(environ, I8X16, builder);
2369 let a_high = builder.ins().uwiden_high(a);
2370 let b_high = builder.ins().uwiden_high(b);
2371 environ.stacks.push1(builder.ins().imul(a_high, b_high));
2372 }
2373 Operator::I32x4ExtMulLowI16x8S => {
2374 let (a, b) = pop2_with_bitcast(environ, I16X8, builder);
2375 let a_low = builder.ins().swiden_low(a);
2376 let b_low = builder.ins().swiden_low(b);
2377 environ.stacks.push1(builder.ins().imul(a_low, b_low));
2378 }
2379 Operator::I32x4ExtMulHighI16x8S => {
2380 let (a, b) = pop2_with_bitcast(environ, I16X8, builder);
2381 let a_high = builder.ins().swiden_high(a);
2382 let b_high = builder.ins().swiden_high(b);
2383 environ.stacks.push1(builder.ins().imul(a_high, b_high));
2384 }
2385 Operator::I32x4ExtMulLowI16x8U => {
2386 let (a, b) = pop2_with_bitcast(environ, I16X8, builder);
2387 let a_low = builder.ins().uwiden_low(a);
2388 let b_low = builder.ins().uwiden_low(b);
2389 environ.stacks.push1(builder.ins().imul(a_low, b_low));
2390 }
2391 Operator::I32x4ExtMulHighI16x8U => {
2392 let (a, b) = pop2_with_bitcast(environ, I16X8, builder);
2393 let a_high = builder.ins().uwiden_high(a);
2394 let b_high = builder.ins().uwiden_high(b);
2395 environ.stacks.push1(builder.ins().imul(a_high, b_high));
2396 }
2397 Operator::I64x2ExtMulLowI32x4S => {
2398 let (a, b) = pop2_with_bitcast(environ, I32X4, builder);
2399 let a_low = builder.ins().swiden_low(a);
2400 let b_low = builder.ins().swiden_low(b);
2401 environ.stacks.push1(builder.ins().imul(a_low, b_low));
2402 }
2403 Operator::I64x2ExtMulHighI32x4S => {
2404 let (a, b) = pop2_with_bitcast(environ, I32X4, builder);
2405 let a_high = builder.ins().swiden_high(a);
2406 let b_high = builder.ins().swiden_high(b);
2407 environ.stacks.push1(builder.ins().imul(a_high, b_high));
2408 }
2409 Operator::I64x2ExtMulLowI32x4U => {
2410 let (a, b) = pop2_with_bitcast(environ, I32X4, builder);
2411 let a_low = builder.ins().uwiden_low(a);
2412 let b_low = builder.ins().uwiden_low(b);
2413 environ.stacks.push1(builder.ins().imul(a_low, b_low));
2414 }
2415 Operator::I64x2ExtMulHighI32x4U => {
2416 let (a, b) = pop2_with_bitcast(environ, I32X4, builder);
2417 let a_high = builder.ins().uwiden_high(a);
2418 let b_high = builder.ins().uwiden_high(b);
2419 environ.stacks.push1(builder.ins().imul(a_high, b_high));
2420 }
2421 Operator::MemoryDiscard { .. } => {
2422 return Err(wasm_unsupported!(
2423 "proposed memory-control operator {:?}",
2424 op
2425 ));
2426 }
2427
2428 Operator::F32x4RelaxedMax | Operator::F64x2RelaxedMax => {
2429 let ty = type_of(op);
2430 let (a, b) = pop2_with_bitcast(environ, ty, builder);
2431 environ.stacks.push1(
2432 if environ.relaxed_simd_deterministic() || !environ.is_x86() {
2433 // Deterministic semantics match the `fmax` instruction, or
2434 // the `fAAxBB.max` wasm instruction.
2435 builder.ins().fmax(a, b)
2436 } else {
2437 // Note that this matches the `pmax` translation which has
2438 // careful ordering of its operands to trigger
2439 // pattern-matches in the x86 backend.
2440 let cmp = builder.ins().fcmp(FloatCC::LessThan, a, b);
2441 let cmp = optionally_bitcast_vector(cmp, ty, builder);
2442 builder.ins().bitselect(cmp, b, a)
2443 },
2444 )
2445 }
2446
2447 Operator::F32x4RelaxedMin | Operator::F64x2RelaxedMin => {
2448 let ty = type_of(op);
2449 let (a, b) = pop2_with_bitcast(environ, ty, builder);
2450 environ.stacks.push1(
2451 if environ.relaxed_simd_deterministic() || !environ.is_x86() {
2452 // Deterministic semantics match the `fmin` instruction, or
2453 // the `fAAxBB.min` wasm instruction.
2454 builder.ins().fmin(a, b)
2455 } else {
2456 // Note that this matches the `pmin` translation which has
2457 // careful ordering of its operands to trigger
2458 // pattern-matches in the x86 backend.
2459 let cmp = builder.ins().fcmp(FloatCC::LessThan, b, a);
2460 let cmp = optionally_bitcast_vector(cmp, ty, builder);
2461 builder.ins().bitselect(cmp, b, a)
2462 },
2463 );
2464 }
2465
2466 Operator::I8x16RelaxedSwizzle => {
2467 let (a, b) = pop2_with_bitcast(environ, I8X16, builder);
2468 let result = environ.relaxed_swizzle(builder, a, b);
2469 environ.stacks.push1(result);
2470 }
2471
2472 Operator::F32x4RelaxedMadd => {
2473 let (a, b, c) = pop3_with_bitcast(environ, type_of(op), builder);
2474 let result = environ.fma_f32x4(builder, a, b, c);
2475 environ.stacks.push1(result);
2476 }
2477 Operator::F64x2RelaxedMadd => {
2478 let (a, b, c) = pop3_with_bitcast(environ, type_of(op), builder);
2479 let result = environ.fma_f64x2(builder, a, b, c);
2480 environ.stacks.push1(result);
2481 }
2482 Operator::F32x4RelaxedNmadd => {
2483 let (a, b, c) = pop3_with_bitcast(environ, type_of(op), builder);
2484 let a = builder.ins().fneg(a);
2485 let result = environ.fma_f32x4(builder, a, b, c);
2486 environ.stacks.push1(result);
2487 }
2488 Operator::F64x2RelaxedNmadd => {
2489 let (a, b, c) = pop3_with_bitcast(environ, type_of(op), builder);
2490 let a = builder.ins().fneg(a);
2491 let result = environ.fma_f64x2(builder, a, b, c);
2492 environ.stacks.push1(result);
2493 }
2494
2495 Operator::I8x16RelaxedLaneselect
2496 | Operator::I16x8RelaxedLaneselect
2497 | Operator::I32x4RelaxedLaneselect
2498 | Operator::I64x2RelaxedLaneselect => {
2499 let ty = type_of(op);
2500 let (a, b, c) = pop3_with_bitcast(environ, ty, builder);
2501 // Note that the variable swaps here are intentional due to
2502 // the difference of the order of the wasm op and the clif
2503 // op.
2504 environ.stacks.push1(
2505 if environ.relaxed_simd_deterministic()
2506 || !environ.use_blendv_for_relaxed_laneselect(ty)
2507 {
2508 // Deterministic semantics are a `bitselect` along the lines
2509 // of the wasm `v128.bitselect` instruction.
2510 builder.ins().bitselect(c, a, b)
2511 } else {
2512 builder.ins().blendv(c, a, b)
2513 },
2514 );
2515 }
2516
2517 Operator::I32x4RelaxedTruncF32x4S => {
2518 let a = pop1_with_bitcast(environ, F32X4, builder);
2519 environ.stacks.push1(
2520 if environ.relaxed_simd_deterministic() || !environ.is_x86() {
2521 // Deterministic semantics are to match the
2522 // `i32x4.trunc_sat_f32x4_s` instruction.
2523 builder.ins().fcvt_to_sint_sat(I32X4, a)
2524 } else {
2525 builder.ins().x86_cvtt2dq(I32X4, a)
2526 },
2527 )
2528 }
2529 Operator::I32x4RelaxedTruncF64x2SZero => {
2530 let a = pop1_with_bitcast(environ, F64X2, builder);
2531 let converted_a = if environ.relaxed_simd_deterministic() || !environ.is_x86() {
2532 // Deterministic semantics are to match the
2533 // `i32x4.trunc_sat_f64x2_s_zero` instruction.
2534 builder.ins().fcvt_to_sint_sat(I64X2, a)
2535 } else {
2536 builder.ins().x86_cvtt2dq(I64X2, a)
2537 };
2538 let handle = builder.func.dfg.constants.insert(vec![0u8; 16].into());
2539 let zero = builder.ins().vconst(I64X2, handle);
2540
2541 environ
2542 .stacks
2543 .push1(builder.ins().snarrow(converted_a, zero));
2544 }
2545 Operator::I16x8RelaxedQ15mulrS => {
2546 let (a, b) = pop2_with_bitcast(environ, I16X8, builder);
2547 environ.stacks.push1(
2548 if environ.relaxed_simd_deterministic()
2549 || !environ.use_x86_pmulhrsw_for_relaxed_q15mul()
2550 {
2551 // Deterministic semantics are to match the
2552 // `i16x8.q15mulr_sat_s` instruction.
2553 builder.ins().sqmul_round_sat(a, b)
2554 } else {
2555 builder.ins().x86_pmulhrsw(a, b)
2556 },
2557 );
2558 }
2559 Operator::I16x8RelaxedDotI8x16I7x16S => {
2560 let (a, b) = pop2_with_bitcast(environ, I8X16, builder);
2561 environ.stacks.push1(
2562 if environ.relaxed_simd_deterministic() || !environ.use_x86_pmaddubsw_for_dot() {
2563 // Deterministic semantics are to treat both operands as
2564 // signed integers and perform the dot product.
2565 let alo = builder.ins().swiden_low(a);
2566 let blo = builder.ins().swiden_low(b);
2567 let lo = builder.ins().imul(alo, blo);
2568 let ahi = builder.ins().swiden_high(a);
2569 let bhi = builder.ins().swiden_high(b);
2570 let hi = builder.ins().imul(ahi, bhi);
2571 builder.ins().iadd_pairwise(lo, hi)
2572 } else {
2573 builder.ins().x86_pmaddubsw(a, b)
2574 },
2575 );
2576 }
2577
2578 Operator::I32x4RelaxedDotI8x16I7x16AddS => {
2579 let c = pop1_with_bitcast(environ, I32X4, builder);
2580 let (a, b) = pop2_with_bitcast(environ, I8X16, builder);
2581 let dot =
2582 if environ.relaxed_simd_deterministic() || !environ.use_x86_pmaddubsw_for_dot() {
2583 // Deterministic semantics are to treat both operands as
2584 // signed integers and perform the dot product.
2585 let alo = builder.ins().swiden_low(a);
2586 let blo = builder.ins().swiden_low(b);
2587 let lo = builder.ins().imul(alo, blo);
2588 let ahi = builder.ins().swiden_high(a);
2589 let bhi = builder.ins().swiden_high(b);
2590 let hi = builder.ins().imul(ahi, bhi);
2591 builder.ins().iadd_pairwise(lo, hi)
2592 } else {
2593 builder.ins().x86_pmaddubsw(a, b)
2594 };
2595 let dotlo = builder.ins().swiden_low(dot);
2596 let dothi = builder.ins().swiden_high(dot);
2597 let dot32 = builder.ins().iadd_pairwise(dotlo, dothi);
2598 environ.stacks.push1(builder.ins().iadd(dot32, c));
2599 }
2600
2601 Operator::BrOnNull { relative_depth } => {
2602 let r = environ.stacks.pop1();
2603 let &[.., WasmValType::Ref(r_ty)] = operand_types else {
2604 unreachable!("validation")
2605 };
2606 let is_null = environ.translate_ref_is_null(builder.cursor(), r, r_ty)?;
2607 let (br_destination, inputs) = translate_br_if_args(*relative_depth, environ);
2608 let else_block = builder.create_block();
2609 canonicalise_brif(builder, is_null, br_destination, inputs, else_block, &[]);
2610
2611 builder.seal_block(else_block); // The only predecessor is the current block.
2612 builder.switch_to_block(else_block);
2613 environ.stacks.push1(r);
2614 }
2615 Operator::BrOnNonNull { relative_depth } => {
2616 // We write this a bit differently from the spec to avoid an extra
2617 // block/branch and the typed accounting thereof. Instead of the
2618 // spec's approach, it's described as such:
2619 // Peek the value val from the stack.
2620 // If val is ref.null ht, then: pop the value val from the stack.
2621 // Else: Execute the instruction (br relative_depth).
2622 let r = environ.stacks.peek1();
2623 let [.., WasmValType::Ref(r_ty)] = operand_types else {
2624 unreachable!("validation")
2625 };
2626 let r_ty = *r_ty;
2627 let (br_destination, inputs) = translate_br_if_args(*relative_depth, environ);
2628 let inputs = inputs.to_vec();
2629 let is_null = environ.translate_ref_is_null(builder.cursor(), r, r_ty)?;
2630 let else_block = builder.create_block();
2631 canonicalise_brif(builder, is_null, else_block, &[], br_destination, &inputs);
2632
2633 // In the null case, pop the ref
2634 environ.stacks.pop1();
2635
2636 builder.seal_block(else_block); // The only predecessor is the current block.
2637
2638 // The rest of the translation operates on our is null case, which is
2639 // currently an empty block
2640 builder.switch_to_block(else_block);
2641 }
2642 Operator::CallRef { type_index } => {
2643 // Get function signature
2644 // `index` is the index of the function's signature and `table_index` is the index of
2645 // the table to search the function in.
2646 let type_index = TypeIndex::from_u32(*type_index);
2647 let sigref = environ.get_or_create_sig_ref(builder.func, type_index);
2648 let num_args = environ.num_params_for_function_type(type_index);
2649 let callee = environ.stacks.pop1();
2650
2651 // Bitcast any vector arguments to their default type, I8X16, before calling.
2652 let mut args = environ.stacks.peekn(num_args).to_vec();
2653 bitcast_wasm_params(environ, sigref, &mut args, builder);
2654
2655 let inst_results =
2656 environ.translate_call_ref(builder, environ.next_srcloc, sigref, callee, &args)?;
2657
2658 debug_assert_eq!(
2659 inst_results.len(),
2660 builder.func.dfg.signatures[sigref].returns.len(),
2661 "translate_call_ref results should match the call signature"
2662 );
2663 environ.stacks.popn(num_args);
2664 environ.stacks.pushn(&inst_results);
2665 }
2666 Operator::RefAsNonNull => {
2667 let r = environ.stacks.pop1();
2668 let [.., WasmValType::Ref(r_ty)] = operand_types else {
2669 unreachable!("validation")
2670 };
2671 let is_null = environ.translate_ref_is_null(builder.cursor(), r, *r_ty)?;
2672 environ.trapnz(builder, is_null, crate::TRAP_NULL_REFERENCE);
2673 environ.stacks.push1(r);
2674 }
2675
2676 Operator::RefI31 => {
2677 let val = environ.stacks.pop1();
2678 let i31ref = environ.translate_ref_i31(builder.cursor(), val)?;
2679 environ.stacks.push1(i31ref);
2680 }
2681 Operator::I31GetS => {
2682 let i31ref = environ.stacks.pop1();
2683 let val = environ.translate_i31_get_s(builder, i31ref)?;
2684 environ.stacks.push1(val);
2685 }
2686 Operator::I31GetU => {
2687 let i31ref = environ.stacks.pop1();
2688 let val = environ.translate_i31_get_u(builder, i31ref)?;
2689 environ.stacks.push1(val);
2690 }
2691
2692 Operator::StructNew { struct_type_index } => {
2693 let struct_type_index = TypeIndex::from_u32(*struct_type_index);
2694 let arity = environ.struct_fields_len(struct_type_index)?;
2695 let fields: StructFieldsVec = environ.stacks.peekn(arity).iter().copied().collect();
2696 environ.stacks.popn(arity);
2697 let struct_ref = environ.translate_struct_new(builder, struct_type_index, fields)?;
2698 environ.stacks.push1(struct_ref);
2699 }
2700
2701 Operator::StructNewDefault { struct_type_index } => {
2702 let struct_type_index = TypeIndex::from_u32(*struct_type_index);
2703 let struct_ref = environ.translate_struct_new_default(builder, struct_type_index)?;
2704 environ.stacks.push1(struct_ref);
2705 }
2706
2707 Operator::StructSet {
2708 struct_type_index,
2709 field_index,
2710 } => {
2711 let struct_type_index = TypeIndex::from_u32(*struct_type_index);
2712 let val = environ.stacks.pop1();
2713 let struct_ref = environ.stacks.pop1();
2714 environ.translate_struct_set(
2715 builder,
2716 struct_type_index,
2717 *field_index,
2718 struct_ref,
2719 val,
2720 )?;
2721 }
2722
2723 Operator::StructGetS {
2724 struct_type_index,
2725 field_index,
2726 } => {
2727 let struct_type_index = TypeIndex::from_u32(*struct_type_index);
2728 let struct_ref = environ.stacks.pop1();
2729 let val = environ.translate_struct_get(
2730 builder,
2731 struct_type_index,
2732 *field_index,
2733 struct_ref,
2734 Some(Extension::Sign),
2735 )?;
2736 environ.stacks.push1(val);
2737 }
2738
2739 Operator::StructGetU {
2740 struct_type_index,
2741 field_index,
2742 } => {
2743 let struct_type_index = TypeIndex::from_u32(*struct_type_index);
2744 let struct_ref = environ.stacks.pop1();
2745 let val = environ.translate_struct_get(
2746 builder,
2747 struct_type_index,
2748 *field_index,
2749 struct_ref,
2750 Some(Extension::Zero),
2751 )?;
2752 environ.stacks.push1(val);
2753 }
2754
2755 Operator::StructGet {
2756 struct_type_index,
2757 field_index,
2758 } => {
2759 let struct_type_index = TypeIndex::from_u32(*struct_type_index);
2760 let struct_ref = environ.stacks.pop1();
2761 let val = environ.translate_struct_get(
2762 builder,
2763 struct_type_index,
2764 *field_index,
2765 struct_ref,
2766 None,
2767 )?;
2768 environ.stacks.push1(val);
2769 }
2770
2771 Operator::ArrayNew { array_type_index } => {
2772 let array_type_index = TypeIndex::from_u32(*array_type_index);
2773 let (elem, len) = environ.stacks.pop2();
2774 let array_ref = environ.translate_array_new(builder, array_type_index, elem, len)?;
2775 environ.stacks.push1(array_ref);
2776 }
2777 Operator::ArrayNewDefault { array_type_index } => {
2778 let array_type_index = TypeIndex::from_u32(*array_type_index);
2779 let len = environ.stacks.pop1();
2780 let array_ref = environ.translate_array_new_default(builder, array_type_index, len)?;
2781 environ.stacks.push1(array_ref);
2782 }
2783 Operator::ArrayNewFixed {
2784 array_type_index,
2785 array_size,
2786 } => {
2787 let array_type_index = TypeIndex::from_u32(*array_type_index);
2788 let array_size = usize::try_from(*array_size).unwrap();
2789 let elems = environ.stacks.peekn(array_size).to_vec();
2790 let array_ref = environ.translate_array_new_fixed(builder, array_type_index, &elems)?;
2791 environ.stacks.popn(array_size);
2792 environ.stacks.push1(array_ref);
2793 }
2794 Operator::ArrayNewData {
2795 array_type_index,
2796 array_data_index,
2797 } => {
2798 let array_type_index = TypeIndex::from_u32(*array_type_index);
2799 let array_data_index = DataIndex::from_u32(*array_data_index);
2800 let (data_offset, len) = environ.stacks.pop2();
2801 let array_ref = environ.translate_array_new_data(
2802 builder,
2803 array_type_index,
2804 array_data_index,
2805 data_offset,
2806 len,
2807 )?;
2808 environ.stacks.push1(array_ref);
2809 }
2810 Operator::ArrayNewElem {
2811 array_type_index,
2812 array_elem_index,
2813 } => {
2814 let array_type_index = TypeIndex::from_u32(*array_type_index);
2815 let array_elem_index = ElemIndex::from_u32(*array_elem_index);
2816 let (elem_offset, len) = environ.stacks.pop2();
2817 let array_ref = environ.translate_array_new_elem(
2818 builder,
2819 array_type_index,
2820 array_elem_index,
2821 elem_offset,
2822 len,
2823 )?;
2824 environ.stacks.push1(array_ref);
2825 }
2826 Operator::ArrayCopy {
2827 array_type_index_dst,
2828 array_type_index_src,
2829 } => {
2830 let array_type_index_dst = TypeIndex::from_u32(*array_type_index_dst);
2831 let array_type_index_src = TypeIndex::from_u32(*array_type_index_src);
2832 let (dst_array, dst_index, src_array, src_index, len) = environ.stacks.pop5();
2833 environ.translate_array_copy(
2834 builder,
2835 array_type_index_dst,
2836 dst_array,
2837 dst_index,
2838 array_type_index_src,
2839 src_array,
2840 src_index,
2841 len,
2842 )?;
2843 }
2844 Operator::ArrayFill { array_type_index } => {
2845 let array_type_index = TypeIndex::from_u32(*array_type_index);
2846 let (array, index, val, len) = environ.stacks.pop4();
2847 environ.translate_array_fill(builder, array_type_index, array, index, val, len)?;
2848 }
2849 Operator::ArrayInitData {
2850 array_type_index,
2851 array_data_index,
2852 } => {
2853 let array_type_index = TypeIndex::from_u32(*array_type_index);
2854 let array_data_index = DataIndex::from_u32(*array_data_index);
2855 let (array, dst_index, src_index, len) = environ.stacks.pop4();
2856 environ.translate_array_init_data(
2857 builder,
2858 array_type_index,
2859 array,
2860 dst_index,
2861 array_data_index,
2862 src_index,
2863 len,
2864 )?;
2865 }
2866 Operator::ArrayInitElem {
2867 array_type_index,
2868 array_elem_index,
2869 } => {
2870 let array_type_index = TypeIndex::from_u32(*array_type_index);
2871 let array_elem_index = ElemIndex::from_u32(*array_elem_index);
2872 let (array, dst_index, src_index, len) = environ.stacks.pop4();
2873 environ.translate_array_init_elem(
2874 builder,
2875 array_type_index,
2876 array,
2877 dst_index,
2878 array_elem_index,
2879 src_index,
2880 len,
2881 )?;
2882 }
2883 Operator::ArrayLen => {
2884 let array = environ.stacks.pop1();
2885 let len = environ.translate_array_len(builder, array)?;
2886 environ.stacks.push1(len);
2887 }
2888 Operator::ArrayGet { array_type_index } => {
2889 let array_type_index = TypeIndex::from_u32(*array_type_index);
2890 let (array, index) = environ.stacks.pop2();
2891 let elem =
2892 environ.translate_array_get(builder, array_type_index, array, index, None)?;
2893 environ.stacks.push1(elem);
2894 }
2895 Operator::ArrayGetS { array_type_index } => {
2896 let array_type_index = TypeIndex::from_u32(*array_type_index);
2897 let (array, index) = environ.stacks.pop2();
2898 let elem = environ.translate_array_get(
2899 builder,
2900 array_type_index,
2901 array,
2902 index,
2903 Some(Extension::Sign),
2904 )?;
2905 environ.stacks.push1(elem);
2906 }
2907 Operator::ArrayGetU { array_type_index } => {
2908 let array_type_index = TypeIndex::from_u32(*array_type_index);
2909 let (array, index) = environ.stacks.pop2();
2910 let elem = environ.translate_array_get(
2911 builder,
2912 array_type_index,
2913 array,
2914 index,
2915 Some(Extension::Zero),
2916 )?;
2917 environ.stacks.push1(elem);
2918 }
2919 Operator::ArraySet { array_type_index } => {
2920 let array_type_index = TypeIndex::from_u32(*array_type_index);
2921 let (array, index, elem) = environ.stacks.pop3();
2922 environ.translate_array_set(builder, array_type_index, array, index, elem)?;
2923 }
2924 Operator::RefEq => {
2925 let (r1, r2) = environ.stacks.pop2();
2926 let eq = builder.ins().icmp(ir::condcodes::IntCC::Equal, r1, r2);
2927 let eq = builder.ins().uextend(ir::types::I32, eq);
2928 environ.stacks.push1(eq);
2929 }
2930 Operator::RefTestNonNull { hty } => {
2931 let r = environ.stacks.pop1();
2932 let [.., WasmValType::Ref(r_ty)] = operand_types else {
2933 unreachable!("validation")
2934 };
2935 let heap_type = environ.convert_heap_type(*hty)?;
2936 let result = environ.translate_ref_test(
2937 builder,
2938 WasmRefType {
2939 heap_type,
2940 nullable: false,
2941 },
2942 r,
2943 *r_ty,
2944 )?;
2945 environ.stacks.push1(result);
2946 }
2947 Operator::RefTestNullable { hty } => {
2948 let r = environ.stacks.pop1();
2949 let [.., WasmValType::Ref(r_ty)] = operand_types else {
2950 unreachable!("validation")
2951 };
2952 let heap_type = environ.convert_heap_type(*hty)?;
2953 let result = environ.translate_ref_test(
2954 builder,
2955 WasmRefType {
2956 heap_type,
2957 nullable: true,
2958 },
2959 r,
2960 *r_ty,
2961 )?;
2962 environ.stacks.push1(result);
2963 }
2964 Operator::RefCastNonNull { hty } => {
2965 let r = environ.stacks.pop1();
2966 let [.., WasmValType::Ref(r_ty)] = operand_types else {
2967 unreachable!("validation")
2968 };
2969 let heap_type = environ.convert_heap_type(*hty)?;
2970 let cast_okay = environ.translate_ref_test(
2971 builder,
2972 WasmRefType {
2973 heap_type,
2974 nullable: false,
2975 },
2976 r,
2977 *r_ty,
2978 )?;
2979 environ.trapz(builder, cast_okay, crate::TRAP_CAST_FAILURE);
2980 environ.stacks.push1(r);
2981 }
2982 Operator::RefCastNullable { hty } => {
2983 let r = environ.stacks.pop1();
2984 let [.., WasmValType::Ref(r_ty)] = operand_types else {
2985 unreachable!("validation")
2986 };
2987 let heap_type = environ.convert_heap_type(*hty)?;
2988 let cast_okay = environ.translate_ref_test(
2989 builder,
2990 WasmRefType {
2991 heap_type,
2992 nullable: true,
2993 },
2994 r,
2995 *r_ty,
2996 )?;
2997 environ.trapz(builder, cast_okay, crate::TRAP_CAST_FAILURE);
2998 environ.stacks.push1(r);
2999 }
3000 Operator::BrOnCast {
3001 relative_depth,
3002 to_ref_type,
3003 from_ref_type: _,
3004 } => {
3005 let r = environ.stacks.peek1();
3006 let [.., WasmValType::Ref(r_ty)] = operand_types else {
3007 unreachable!("validation")
3008 };
3009
3010 let to_ref_type = environ.convert_ref_type(*to_ref_type)?;
3011 let cast_is_okay = environ.translate_ref_test(builder, to_ref_type, r, *r_ty)?;
3012
3013 let (cast_succeeds_block, inputs) = translate_br_if_args(*relative_depth, environ);
3014 let cast_fails_block = builder.create_block();
3015 canonicalise_brif(
3016 builder,
3017 cast_is_okay,
3018 cast_succeeds_block,
3019 inputs,
3020 cast_fails_block,
3021 &[
3022 // NB: the `cast_fails_block` is dominated by the current
3023 // block, and therefore doesn't need any block params.
3024 ],
3025 );
3026
3027 // The only predecessor is the current block.
3028 builder.seal_block(cast_fails_block);
3029
3030 // The next Wasm instruction is executed when the cast failed and we
3031 // did not branch away.
3032 builder.switch_to_block(cast_fails_block);
3033 }
3034 Operator::BrOnCastFail {
3035 relative_depth,
3036 to_ref_type,
3037 from_ref_type: _,
3038 } => {
3039 let r = environ.stacks.peek1();
3040 let [.., WasmValType::Ref(r_ty)] = operand_types else {
3041 unreachable!("validation")
3042 };
3043
3044 let to_ref_type = environ.convert_ref_type(*to_ref_type)?;
3045 let cast_is_okay = environ.translate_ref_test(builder, to_ref_type, r, *r_ty)?;
3046
3047 let (cast_fails_block, inputs) = translate_br_if_args(*relative_depth, environ);
3048 let cast_succeeds_block = builder.create_block();
3049 canonicalise_brif(
3050 builder,
3051 cast_is_okay,
3052 cast_succeeds_block,
3053 &[
3054 // NB: the `cast_succeeds_block` is dominated by the current
3055 // block, and therefore doesn't need any block params.
3056 ],
3057 cast_fails_block,
3058 inputs,
3059 );
3060
3061 // The only predecessor is the current block.
3062 builder.seal_block(cast_succeeds_block);
3063
3064 // The next Wasm instruction is executed when the cast succeeded and
3065 // we did not branch away.
3066 builder.switch_to_block(cast_succeeds_block);
3067 }
3068
3069 Operator::AnyConvertExtern => {
3070 // Pop an `externref`, push an `anyref`. But they have the same
3071 // representation, so we don't actually need to do anything.
3072 }
3073 Operator::ExternConvertAny => {
3074 // Pop an `anyref`, push an `externref`. But they have the same
3075 // representation, so we don't actually need to do anything.
3076 }
3077
3078 Operator::ContNew { cont_type_index } => {
3079 let cont_type_index = TypeIndex::from_u32(*cont_type_index);
3080 let arg_types: SmallVec<[_; 8]> = environ
3081 .continuation_arguments(cont_type_index)
3082 .to_smallvec();
3083 let result_types: SmallVec<[_; 8]> =
3084 environ.continuation_returns(cont_type_index).to_smallvec();
3085 let r = environ.stacks.pop1();
3086 let contobj = environ.translate_cont_new(builder, r, &arg_types, &result_types)?;
3087 environ.stacks.push1(contobj);
3088 }
3089 Operator::ContBind {
3090 argument_index,
3091 result_index,
3092 } => {
3093 let src_types = environ.continuation_arguments(TypeIndex::from_u32(*argument_index));
3094 let dst_arity = environ
3095 .continuation_arguments(TypeIndex::from_u32(*result_index))
3096 .len();
3097 let arg_count = src_types.len() - dst_arity;
3098
3099 let arg_types = &src_types[0..arg_count];
3100 for arg_type in arg_types {
3101 // We can't bind GC objects using cont.bind at the moment: We
3102 // don't have the necessary infrastructure to traverse the
3103 // buffers used by cont.bind when looking for GC roots. Thus,
3104 // this crude check ensures that these buffers can never contain
3105 // GC roots to begin with.
3106 if arg_type.is_vmgcref_type_and_not_i31() {
3107 return Err(wasmtime_environ::WasmError::Unsupported(
3108 "cont.bind does not support GC types at the moment".into(),
3109 ));
3110 }
3111 }
3112
3113 let (original_contobj, args) =
3114 environ.stacks.peekn(arg_count + 1).split_last().unwrap();
3115 let original_contobj = *original_contobj;
3116 let args = args.to_vec();
3117
3118 let new_contobj = environ.translate_cont_bind(builder, original_contobj, &args);
3119
3120 environ.stacks.popn(arg_count + 1);
3121 environ.stacks.push1(new_contobj);
3122 }
3123 Operator::Suspend { tag_index } => {
3124 let tag_index = TagIndex::from_u32(*tag_index);
3125 let param_types = environ.tag_params(tag_index).to_vec();
3126 let return_types: SmallVec<[_; 8]> = environ
3127 .tag_returns(tag_index)
3128 .iter()
3129 .map(|ty| crate::value_type(environ.isa(), *ty))
3130 .collect();
3131
3132 let params = environ.stacks.peekn(param_types.len()).to_vec();
3133 let param_count = params.len();
3134
3135 let return_values =
3136 environ.translate_suspend(builder, tag_index.as_u32(), ¶ms, &return_types);
3137
3138 environ.stacks.popn(param_count);
3139 environ.stacks.pushn(&return_values);
3140 }
3141 Operator::Resume {
3142 cont_type_index,
3143 resume_table: wasm_resume_table,
3144 } => {
3145 // We translate the block indices in the wasm resume_table to actual Blocks.
3146 let mut clif_resume_table = vec![];
3147 for handle in &wasm_resume_table.handlers {
3148 match handle {
3149 wasmparser::Handle::OnLabel { tag, label } => {
3150 let i = environ.stacks.control_stack.len() - 1 - (*label as usize);
3151 let frame = &mut environ.stacks.control_stack[i];
3152 // This is side-effecting!
3153 frame.set_branched_to_exit();
3154 clif_resume_table.push((*tag, Some(frame.br_destination())));
3155 }
3156 wasmparser::Handle::OnSwitch { tag } => {
3157 clif_resume_table.push((*tag, None));
3158 }
3159 }
3160 }
3161
3162 let cont_type_index = TypeIndex::from_u32(*cont_type_index);
3163 let arity = environ.continuation_arguments(cont_type_index).len();
3164 let (contobj, call_args) = environ.stacks.peekn(arity + 1).split_last().unwrap();
3165 let contobj = *contobj;
3166 let call_args = call_args.to_vec();
3167
3168 let cont_return_vals = environ.translate_resume(
3169 builder,
3170 cont_type_index.as_u32(),
3171 contobj,
3172 &call_args,
3173 &clif_resume_table,
3174 )?;
3175
3176 environ.stacks.popn(arity + 1); // arguments + continuation
3177 environ.stacks.pushn(&cont_return_vals);
3178 }
3179 Operator::ResumeThrow {
3180 cont_type_index: _,
3181 tag_index: _,
3182 resume_table: _,
3183 } => {
3184 // TODO(10248) This depends on exception handling
3185 return Err(wasmtime_environ::WasmError::Unsupported(
3186 "resume.throw instructions not supported, yet".to_string(),
3187 ));
3188 }
3189 Operator::Switch {
3190 cont_type_index,
3191 tag_index,
3192 } => {
3193 // Arguments of the continuation we are going to switch to
3194 let continuation_argument_types: SmallVec<[_; 8]> = environ
3195 .continuation_arguments(TypeIndex::from_u32(*cont_type_index))
3196 .to_smallvec();
3197 // Arity includes the continuation argument
3198 let arity = continuation_argument_types.len();
3199 let (contobj, switch_args) = environ.stacks.peekn(arity).split_last().unwrap();
3200 let contobj = *contobj;
3201 let switch_args = switch_args.to_vec();
3202
3203 // Type of the continuation we are going to create by suspending the
3204 // currently running stack
3205 let current_continuation_type = continuation_argument_types.last().unwrap();
3206 let current_continuation_type = current_continuation_type.unwrap_ref_type();
3207
3208 // Argument types of current_continuation_type. These will in turn
3209 // be the types of the arguments we receive when someone switches
3210 // back to this switch instruction
3211 let current_continuation_arg_types: SmallVec<[_; 8]> =
3212 match current_continuation_type.heap_type {
3213 WasmHeapType::ConcreteCont(index) => {
3214 let mti = index
3215 .as_module_type_index()
3216 .expect("Only supporting module type indices on switch for now");
3217
3218 environ
3219 .continuation_arguments(TypeIndex::from_u32(mti.as_u32()))
3220 .iter()
3221 .map(|ty| crate::value_type(environ.isa(), *ty))
3222 .collect()
3223 }
3224 _ => panic!("Invalid type on switch"),
3225 };
3226
3227 let switch_return_values = environ.translate_switch(
3228 builder,
3229 *tag_index,
3230 contobj,
3231 &switch_args,
3232 ¤t_continuation_arg_types,
3233 )?;
3234
3235 environ.stacks.popn(arity);
3236 environ.stacks.pushn(&switch_return_values)
3237 }
3238
3239 Operator::GlobalAtomicGet { .. }
3240 | Operator::GlobalAtomicSet { .. }
3241 | Operator::GlobalAtomicRmwAdd { .. }
3242 | Operator::GlobalAtomicRmwSub { .. }
3243 | Operator::GlobalAtomicRmwOr { .. }
3244 | Operator::GlobalAtomicRmwXor { .. }
3245 | Operator::GlobalAtomicRmwAnd { .. }
3246 | Operator::GlobalAtomicRmwXchg { .. }
3247 | Operator::GlobalAtomicRmwCmpxchg { .. }
3248 | Operator::TableAtomicGet { .. }
3249 | Operator::TableAtomicSet { .. }
3250 | Operator::TableAtomicRmwXchg { .. }
3251 | Operator::TableAtomicRmwCmpxchg { .. }
3252 | Operator::StructAtomicGet { .. }
3253 | Operator::StructAtomicGetS { .. }
3254 | Operator::StructAtomicGetU { .. }
3255 | Operator::StructAtomicSet { .. }
3256 | Operator::StructAtomicRmwAdd { .. }
3257 | Operator::StructAtomicRmwSub { .. }
3258 | Operator::StructAtomicRmwOr { .. }
3259 | Operator::StructAtomicRmwXor { .. }
3260 | Operator::StructAtomicRmwAnd { .. }
3261 | Operator::StructAtomicRmwXchg { .. }
3262 | Operator::StructAtomicRmwCmpxchg { .. }
3263 | Operator::ArrayAtomicGet { .. }
3264 | Operator::ArrayAtomicGetS { .. }
3265 | Operator::ArrayAtomicGetU { .. }
3266 | Operator::ArrayAtomicSet { .. }
3267 | Operator::ArrayAtomicRmwAdd { .. }
3268 | Operator::ArrayAtomicRmwSub { .. }
3269 | Operator::ArrayAtomicRmwOr { .. }
3270 | Operator::ArrayAtomicRmwXor { .. }
3271 | Operator::ArrayAtomicRmwAnd { .. }
3272 | Operator::ArrayAtomicRmwXchg { .. }
3273 | Operator::ArrayAtomicRmwCmpxchg { .. }
3274 | Operator::RefI31Shared { .. } => {
3275 return Err(wasm_unsupported!(
3276 "shared-everything-threads operators are not yet implemented"
3277 ));
3278 }
3279
3280 Operator::I64MulWideS => {
3281 let (arg1, arg2) = environ.stacks.pop2();
3282 let arg1 = builder.ins().sextend(I128, arg1);
3283 let arg2 = builder.ins().sextend(I128, arg2);
3284 let result = builder.ins().imul(arg1, arg2);
3285 let (lo, hi) = builder.ins().isplit(result);
3286 environ.stacks.push2(lo, hi);
3287 }
3288 Operator::I64MulWideU => {
3289 let (arg1, arg2) = environ.stacks.pop2();
3290 let arg1 = builder.ins().uextend(I128, arg1);
3291 let arg2 = builder.ins().uextend(I128, arg2);
3292 let result = builder.ins().imul(arg1, arg2);
3293 let (lo, hi) = builder.ins().isplit(result);
3294 environ.stacks.push2(lo, hi);
3295 }
3296 Operator::I64Add128 => {
3297 let (arg1, arg2, arg3, arg4) = environ.stacks.pop4();
3298 let arg1 = builder.ins().iconcat(arg1, arg2);
3299 let arg2 = builder.ins().iconcat(arg3, arg4);
3300 let result = builder.ins().iadd(arg1, arg2);
3301 let (res1, res2) = builder.ins().isplit(result);
3302 environ.stacks.push2(res1, res2);
3303 }
3304 Operator::I64Sub128 => {
3305 let (arg1, arg2, arg3, arg4) = environ.stacks.pop4();
3306 let arg1 = builder.ins().iconcat(arg1, arg2);
3307 let arg2 = builder.ins().iconcat(arg3, arg4);
3308 let result = builder.ins().isub(arg1, arg2);
3309 let (res1, res2) = builder.ins().isplit(result);
3310 environ.stacks.push2(res1, res2);
3311 }
3312
3313 // catch-all as `Operator` is `#[non_exhaustive]`
3314 op => return Err(wasm_unsupported!("operator {op:?}")),
3315 };
3316 Ok(())
3317 }
3318
3319 /// Deals with a Wasm instruction located in an unreachable portion of the code. Most of them
3320 /// are dropped but special ones like `End` or `Else` signal the potential end of the unreachable
3321 /// portion so the translation state must be updated accordingly.
translate_unreachable_operator( validator: &FuncValidator<impl WasmModuleResources>, op: &Operator, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>, ) -> WasmResult<()>3322 fn translate_unreachable_operator(
3323 validator: &FuncValidator<impl WasmModuleResources>,
3324 op: &Operator,
3325 builder: &mut FunctionBuilder,
3326 environ: &mut FuncEnvironment<'_>,
3327 ) -> WasmResult<()> {
3328 debug_assert!(!environ.is_reachable());
3329 match *op {
3330 Operator::If { blockty } => {
3331 // Push a placeholder control stack entry. The if isn't reachable,
3332 // so we don't have any branches anywhere.
3333 environ.stacks.push_if(
3334 ir::Block::reserved_value(),
3335 ElseData::NoElse {
3336 branch_inst: ir::Inst::reserved_value(),
3337 placeholder: ir::Block::reserved_value(),
3338 },
3339 0,
3340 0,
3341 blockty,
3342 );
3343 }
3344 Operator::Loop { blockty: _ }
3345 | Operator::Block { blockty: _ }
3346 | Operator::TryTable { try_table: _ } => {
3347 environ.stacks.push_block(ir::Block::reserved_value(), 0, 0);
3348 }
3349 Operator::Else => {
3350 let i = environ.stacks.control_stack.len() - 1;
3351 let reachable = environ.is_reachable();
3352 match environ.stacks.control_stack[i] {
3353 ControlStackFrame::If {
3354 ref else_data,
3355 head_is_reachable,
3356 ref mut consequent_ends_reachable,
3357 blocktype,
3358 ..
3359 } => {
3360 debug_assert!(consequent_ends_reachable.is_none());
3361 *consequent_ends_reachable = Some(reachable);
3362
3363 if head_is_reachable {
3364 // We have a branch from the head of the `if` to the `else`.
3365 environ.stacks.reachable = true;
3366
3367 let else_block = match *else_data {
3368 ElseData::NoElse {
3369 branch_inst,
3370 placeholder,
3371 } => {
3372 let (params, _results) =
3373 blocktype_params_results(validator, blocktype)?;
3374 let else_block = block_with_params(builder, params, environ)?;
3375 let frame = environ.stacks.control_stack.last().unwrap();
3376 frame.truncate_value_stack_to_else_params(
3377 &mut environ.stacks.stack,
3378 &mut environ.stacks.stack_shape,
3379 );
3380
3381 // We change the target of the branch instruction.
3382 builder.change_jump_destination(
3383 branch_inst,
3384 placeholder,
3385 else_block,
3386 );
3387 builder.seal_block(else_block);
3388 else_block
3389 }
3390 ElseData::WithElse { else_block } => {
3391 let frame = environ.stacks.control_stack.last().unwrap();
3392 frame.truncate_value_stack_to_else_params(
3393 &mut environ.stacks.stack,
3394 &mut environ.stacks.stack_shape,
3395 );
3396 else_block
3397 }
3398 };
3399
3400 builder.switch_to_block(else_block);
3401
3402 // Again, no need to push the parameters for the `else`,
3403 // since we already did when we saw the original `if`. See
3404 // the comment for translating `Operator::Else` in
3405 // `translate_operator` for details.
3406 }
3407 }
3408 _ => unreachable!(),
3409 }
3410 }
3411 Operator::End => {
3412 let value_stack = &mut environ.stacks.stack;
3413 let stack_shape = &mut environ.stacks.stack_shape;
3414 let control_stack = &mut environ.stacks.control_stack;
3415 let frame = control_stack.pop().unwrap();
3416
3417 frame.restore_catch_handlers(&mut environ.stacks.handlers, builder);
3418
3419 // Pop unused parameters from stack.
3420 frame.truncate_value_stack_to_original_size(value_stack, stack_shape);
3421
3422 let reachable_anyway = match frame {
3423 // If it is a loop we also have to seal the body loop block
3424 ControlStackFrame::Loop { header, .. } => {
3425 builder.seal_block(header);
3426 // And loops can't have branches to the end.
3427 false
3428 }
3429 // If we never set `consequent_ends_reachable` then that means
3430 // we are finishing the consequent now, and there was no
3431 // `else`. Whether the following block is reachable depends only
3432 // on if the head was reachable.
3433 ControlStackFrame::If {
3434 head_is_reachable,
3435 consequent_ends_reachable: None,
3436 ..
3437 } => head_is_reachable,
3438 // Since we are only in this function when in unreachable code,
3439 // we know that the alternative just ended unreachable. Whether
3440 // the following block is reachable depends on if the consequent
3441 // ended reachable or not.
3442 ControlStackFrame::If {
3443 head_is_reachable,
3444 consequent_ends_reachable: Some(consequent_ends_reachable),
3445 ..
3446 } => head_is_reachable && consequent_ends_reachable,
3447 // All other control constructs are already handled.
3448 _ => false,
3449 };
3450
3451 if frame.exit_is_branched_to() || reachable_anyway {
3452 builder.switch_to_block(frame.following_code());
3453 builder.seal_block(frame.following_code());
3454
3455 // And add the return values of the block but only if the next block is reachable
3456 // (which corresponds to testing if the stack depth is 1)
3457 value_stack.extend_from_slice(builder.block_params(frame.following_code()));
3458 environ.stacks.reachable = true;
3459 }
3460 }
3461 _ => {
3462 // We don't translate because this is unreachable code
3463 }
3464 }
3465
3466 Ok(())
3467 }
3468
3469 /// This function is a generalized helper for validating that a wasm-supplied
3470 /// heap address is in-bounds.
3471 ///
3472 /// This function takes a litany of parameters and requires that the *Wasm*
3473 /// address to be verified is at the top of the stack in `state`. This will
3474 /// generate necessary IR to validate that the heap address is correctly
3475 /// in-bounds, and various parameters are returned describing the valid *native*
3476 /// heap address if execution reaches that point.
3477 ///
3478 /// Returns `None` when the Wasm access will unconditionally trap.
3479 ///
3480 /// Returns `(flags, wasm_addr, native_addr)`.
prepare_addr( memarg: &MemArg, access_size: u8, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>, ) -> WasmResult<Reachability<(MemFlags, Value, Value)>>3481 fn prepare_addr(
3482 memarg: &MemArg,
3483 access_size: u8,
3484 builder: &mut FunctionBuilder,
3485 environ: &mut FuncEnvironment<'_>,
3486 ) -> WasmResult<Reachability<(MemFlags, Value, Value)>> {
3487 let index = environ.stacks.pop1();
3488
3489 let memory_index = MemoryIndex::from_u32(memarg.memory);
3490 let heap = environ.get_or_create_heap(builder.func, memory_index);
3491
3492 // How exactly the bounds check is performed here and what it's performed
3493 // on is a bit tricky. Generally we want to rely on access violations (e.g.
3494 // segfaults) to generate traps since that means we don't have to bounds
3495 // check anything explicitly.
3496 //
3497 // (1) If we don't have a guard page of unmapped memory, though, then we
3498 // can't rely on this trapping behavior through segfaults. Instead we need
3499 // to bounds-check the entire memory access here which is everything from
3500 // `addr32 + offset` to `addr32 + offset + width` (not inclusive). In this
3501 // scenario our adjusted offset that we're checking is `memarg.offset +
3502 // access_size`. Note that we do saturating arithmetic here to avoid
3503 // overflow. The addition here is in the 64-bit space, which means that
3504 // we'll never overflow for 32-bit wasm but for 64-bit this is an issue. If
3505 // our effective offset is u64::MAX though then it's impossible for for
3506 // that to actually be a valid offset because otherwise the wasm linear
3507 // memory would take all of the host memory!
3508 //
3509 // (2) If we have a guard page, however, then we can perform a further
3510 // optimization of the generated code by only checking multiples of the
3511 // offset-guard size to be more CSE-friendly. Knowing that we have at least
3512 // 1 page of a guard page we're then able to disregard the `width` since we
3513 // know it's always less than one page. Our bounds check will be for the
3514 // first byte which will either succeed and be guaranteed to fault if it's
3515 // actually out of bounds, or the bounds check itself will fail. In any case
3516 // we assert that the width is reasonably small for now so this assumption
3517 // can be adjusted in the future if we get larger widths.
3518 //
3519 // Put another way we can say, where `y < offset_guard_size`:
3520 //
3521 // n * offset_guard_size + y = offset
3522 //
3523 // We'll then pass `n * offset_guard_size` as the bounds check value. If
3524 // this traps then our `offset` would have trapped anyway. If this check
3525 // passes we know
3526 //
3527 // addr32 + n * offset_guard_size < bound
3528 //
3529 // which means
3530 //
3531 // addr32 + n * offset_guard_size + y < bound + offset_guard_size
3532 //
3533 // because `y < offset_guard_size`, which then means:
3534 //
3535 // addr32 + offset < bound + offset_guard_size
3536 //
3537 // Since we know that that guard size bytes are all unmapped we're
3538 // guaranteed that `offset` and the `width` bytes after it are either
3539 // in-bounds or will hit the guard page, meaning we'll get the desired
3540 // semantics we want.
3541 //
3542 // ---
3543 //
3544 // With all that in mind remember that the goal is to bounds check as few
3545 // things as possible. To facilitate this the "fast path" is expected to be
3546 // hit like so:
3547 //
3548 // * For wasm32, wasmtime defaults to 4gb "static" memories with 2gb guard
3549 // regions. This means that for all offsets <=2gb, we hit the optimized
3550 // case for `heap_addr` on static memories 4gb in size in cranelift's
3551 // legalization of `heap_addr`, eliding the bounds check entirely.
3552 //
3553 // * For wasm64 offsets <=2gb will generate a single `heap_addr`
3554 // instruction, but at this time all heaps are "dynamic" which means that
3555 // a single bounds check is forced. Ideally we'd do better here, but
3556 // that's the current state of affairs.
3557 //
3558 // Basically we assume that most configurations have a guard page and most
3559 // offsets in `memarg` are <=2gb, which means we get the fast path of one
3560 // `heap_addr` instruction plus a hardcoded i32-offset in memory-related
3561 // instructions.
3562 let heap = environ.heaps()[heap].clone();
3563 let addr = match u32::try_from(memarg.offset) {
3564 // If our offset fits within a u32, then we can place the it into the
3565 // offset immediate of the `heap_addr` instruction.
3566 Ok(offset) => bounds_check_and_compute_addr(
3567 builder,
3568 environ,
3569 &heap,
3570 index,
3571 BoundsCheck::StaticOffset {
3572 offset,
3573 access_size,
3574 },
3575 ir::TrapCode::HEAP_OUT_OF_BOUNDS,
3576 ),
3577
3578 // If the offset doesn't fit within a u32, then we can't pass it
3579 // directly into `heap_addr`.
3580 //
3581 // One reasonable question you might ask is "why not?". There's no
3582 // fundamental reason why `heap_addr` *must* take a 32-bit offset. The
3583 // reason this isn't done, though, is that blindly changing the offset
3584 // to a 64-bit offset increases the size of the `InstructionData` enum
3585 // in cranelift by 8 bytes (16 to 24). This can have significant
3586 // performance implications so the conclusion when this was written was
3587 // that we shouldn't do that.
3588 //
3589 // Without the ability to put the whole offset into the `heap_addr`
3590 // instruction we need to fold the offset into the address itself with
3591 // an unsigned addition. In doing so though we need to check for
3592 // overflow because that would mean the address is out-of-bounds (wasm
3593 // bounds checks happen on the effective 33 or 65 bit address once the
3594 // offset is factored in).
3595 //
3596 // Once we have the effective address, offset already folded in, then
3597 // `heap_addr` is used to verify that the address is indeed in-bounds.
3598 //
3599 // Note that this is generating what's likely to be at least two
3600 // branches, one for the overflow and one for the bounds check itself.
3601 // For now though that should hopefully be ok since 4gb+ offsets are
3602 // relatively odd/rare. In the future if needed we can look into
3603 // optimizing this more.
3604 Err(_) => {
3605 let offset = builder
3606 .ins()
3607 .iconst(heap.index_type(), memarg.offset.cast_signed());
3608 let adjusted_index = environ.uadd_overflow_trap(
3609 builder,
3610 index,
3611 offset,
3612 ir::TrapCode::HEAP_OUT_OF_BOUNDS,
3613 );
3614 bounds_check_and_compute_addr(
3615 builder,
3616 environ,
3617 &heap,
3618 adjusted_index,
3619 BoundsCheck::StaticOffset {
3620 offset: 0,
3621 access_size,
3622 },
3623 ir::TrapCode::HEAP_OUT_OF_BOUNDS,
3624 )
3625 }
3626 };
3627 let addr = match addr {
3628 Reachability::Unreachable => return Ok(Reachability::Unreachable),
3629 Reachability::Reachable(a) => a,
3630 };
3631
3632 // Note that we don't set `is_aligned` here, even if the load instruction's
3633 // alignment immediate may says it's aligned, because WebAssembly's
3634 // immediate field is just a hint, while Cranelift's aligned flag needs a
3635 // guarantee. WebAssembly memory accesses are always little-endian.
3636 let mut flags = MemFlags::new();
3637 flags.set_endianness(ir::Endianness::Little);
3638
3639 // The access occurs to the `heap` disjoint category of abstract
3640 // state. This may allow alias analysis to merge redundant loads,
3641 // etc. when heap accesses occur interleaved with other (table,
3642 // vmctx, stack) accesses.
3643 flags.set_alias_region(Some(ir::AliasRegion::Heap));
3644
3645 Ok(Reachability::Reachable((flags, index, addr)))
3646 }
3647
align_atomic_addr( memarg: &MemArg, loaded_bytes: u8, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>, )3648 fn align_atomic_addr(
3649 memarg: &MemArg,
3650 loaded_bytes: u8,
3651 builder: &mut FunctionBuilder,
3652 environ: &mut FuncEnvironment<'_>,
3653 ) {
3654 // Atomic addresses must all be aligned correctly, and for now we check
3655 // alignment before we check out-of-bounds-ness. The order of this check may
3656 // need to be updated depending on the outcome of the official threads
3657 // proposal itself.
3658 //
3659 // Note that with an offset>0 we generate an `iadd_imm` where the result is
3660 // thrown away after the offset check. This may truncate the offset and the
3661 // result may overflow as well, but those conditions won't affect the
3662 // alignment check itself. This can probably be optimized better and we
3663 // should do so in the future as well.
3664 if loaded_bytes > 1 {
3665 let addr = environ.stacks.peek1();
3666 let effective_addr = if memarg.offset == 0 {
3667 addr
3668 } else {
3669 builder.ins().iadd_imm(addr, memarg.offset.cast_signed())
3670 };
3671 debug_assert!(loaded_bytes.is_power_of_two());
3672 let misalignment = builder
3673 .ins()
3674 .band_imm(effective_addr, i64::from(loaded_bytes - 1));
3675 let f = builder.ins().icmp_imm(IntCC::NotEqual, misalignment, 0);
3676 environ.trapnz(builder, f, crate::TRAP_HEAP_MISALIGNED);
3677 }
3678 }
3679
3680 /// Like `prepare_addr` but for atomic accesses.
3681 ///
3682 /// Returns `None` when the Wasm access will unconditionally trap.
prepare_atomic_addr( memarg: &MemArg, loaded_bytes: u8, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>, ) -> WasmResult<Reachability<(MemFlags, Value, Value)>>3683 fn prepare_atomic_addr(
3684 memarg: &MemArg,
3685 loaded_bytes: u8,
3686 builder: &mut FunctionBuilder,
3687 environ: &mut FuncEnvironment<'_>,
3688 ) -> WasmResult<Reachability<(MemFlags, Value, Value)>> {
3689 align_atomic_addr(memarg, loaded_bytes, builder, environ);
3690 prepare_addr(memarg, loaded_bytes, builder, environ)
3691 }
3692
3693 /// Translate a load instruction.
3694 ///
3695 /// Returns the execution state's reachability after the load is translated.
translate_load( memarg: &MemArg, opcode: ir::Opcode, result_ty: Type, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>, ) -> WasmResult<Reachability<()>>3696 fn translate_load(
3697 memarg: &MemArg,
3698 opcode: ir::Opcode,
3699 result_ty: Type,
3700 builder: &mut FunctionBuilder,
3701 environ: &mut FuncEnvironment<'_>,
3702 ) -> WasmResult<Reachability<()>> {
3703 let mem_op_size = mem_op_size(opcode, result_ty);
3704 let (flags, wasm_index, base) = match prepare_addr(memarg, mem_op_size, builder, environ)? {
3705 Reachability::Unreachable => return Ok(Reachability::Unreachable),
3706 Reachability::Reachable((f, i, b)) => (f, i, b),
3707 };
3708
3709 environ.before_load(builder, mem_op_size, wasm_index, memarg.offset);
3710
3711 let (load, dfg) = builder
3712 .ins()
3713 .Load(opcode, result_ty, flags, Offset32::new(0), base);
3714 environ.stacks.push1(dfg.first_result(load));
3715 Ok(Reachability::Reachable(()))
3716 }
3717
3718 /// Translate a store instruction.
translate_store( memarg: &MemArg, opcode: ir::Opcode, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>, ) -> WasmResult<()>3719 fn translate_store(
3720 memarg: &MemArg,
3721 opcode: ir::Opcode,
3722 builder: &mut FunctionBuilder,
3723 environ: &mut FuncEnvironment<'_>,
3724 ) -> WasmResult<()> {
3725 let val = environ.stacks.pop1();
3726 let val_ty = builder.func.dfg.value_type(val);
3727 let mem_op_size = mem_op_size(opcode, val_ty);
3728
3729 let (flags, wasm_index, base) = unwrap_or_return_unreachable_state!(
3730 environ,
3731 prepare_addr(memarg, mem_op_size, builder, environ)?
3732 );
3733
3734 environ.before_store(builder, mem_op_size, wasm_index, memarg.offset);
3735
3736 builder
3737 .ins()
3738 .Store(opcode, val_ty, flags, Offset32::new(0), val, base);
3739 Ok(())
3740 }
3741
mem_op_size(opcode: ir::Opcode, ty: Type) -> u83742 fn mem_op_size(opcode: ir::Opcode, ty: Type) -> u8 {
3743 match opcode {
3744 ir::Opcode::Istore8 | ir::Opcode::Sload8 | ir::Opcode::Uload8 => 1,
3745 ir::Opcode::Istore16 | ir::Opcode::Sload16 | ir::Opcode::Uload16 => 2,
3746 ir::Opcode::Istore32 | ir::Opcode::Sload32 | ir::Opcode::Uload32 => 4,
3747 ir::Opcode::Store | ir::Opcode::Load => u8::try_from(ty.bytes()).unwrap(),
3748 _ => panic!("unknown size of mem op for {opcode:?}"),
3749 }
3750 }
3751
translate_icmp(cc: IntCC, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>)3752 fn translate_icmp(cc: IntCC, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>) {
3753 let (arg0, arg1) = environ.stacks.pop2();
3754 let val = builder.ins().icmp(cc, arg0, arg1);
3755 environ.stacks.push1(builder.ins().uextend(I32, val));
3756 }
3757
translate_atomic_rmw( widened_ty: Type, access_ty: Type, op: AtomicRmwOp, memarg: &MemArg, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>, ) -> WasmResult<()>3758 fn translate_atomic_rmw(
3759 widened_ty: Type,
3760 access_ty: Type,
3761 op: AtomicRmwOp,
3762 memarg: &MemArg,
3763 builder: &mut FunctionBuilder,
3764 environ: &mut FuncEnvironment<'_>,
3765 ) -> WasmResult<()> {
3766 let mut arg2 = environ.stacks.pop1();
3767 let arg2_ty = builder.func.dfg.value_type(arg2);
3768
3769 // The operation is performed at type `access_ty`, and the old value is zero-extended
3770 // to type `widened_ty`.
3771 match access_ty {
3772 I8 | I16 | I32 | I64 => {}
3773 _ => {
3774 return Err(wasm_unsupported!(
3775 "atomic_rmw: unsupported access type {:?}",
3776 access_ty
3777 ));
3778 }
3779 };
3780 let w_ty_ok = match widened_ty {
3781 I32 | I64 => true,
3782 _ => false,
3783 };
3784 assert!(w_ty_ok && widened_ty.bytes() >= access_ty.bytes());
3785
3786 assert!(arg2_ty.bytes() >= access_ty.bytes());
3787 if arg2_ty.bytes() > access_ty.bytes() {
3788 arg2 = builder.ins().ireduce(access_ty, arg2);
3789 }
3790
3791 let (flags, _, addr) = unwrap_or_return_unreachable_state!(
3792 environ,
3793 prepare_atomic_addr(
3794 memarg,
3795 u8::try_from(access_ty.bytes()).unwrap(),
3796 builder,
3797 environ,
3798 )?
3799 );
3800
3801 let mut res = builder.ins().atomic_rmw(access_ty, flags, op, addr, arg2);
3802 if access_ty != widened_ty {
3803 res = builder.ins().uextend(widened_ty, res);
3804 }
3805 environ.stacks.push1(res);
3806 Ok(())
3807 }
3808
translate_atomic_cas( widened_ty: Type, access_ty: Type, memarg: &MemArg, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>, ) -> WasmResult<()>3809 fn translate_atomic_cas(
3810 widened_ty: Type,
3811 access_ty: Type,
3812 memarg: &MemArg,
3813 builder: &mut FunctionBuilder,
3814 environ: &mut FuncEnvironment<'_>,
3815 ) -> WasmResult<()> {
3816 let (mut expected, mut replacement) = environ.stacks.pop2();
3817 let expected_ty = builder.func.dfg.value_type(expected);
3818 let replacement_ty = builder.func.dfg.value_type(replacement);
3819
3820 // The compare-and-swap is performed at type `access_ty`, and the old value is zero-extended
3821 // to type `widened_ty`.
3822 match access_ty {
3823 I8 | I16 | I32 | I64 => {}
3824 _ => {
3825 return Err(wasm_unsupported!(
3826 "atomic_cas: unsupported access type {:?}",
3827 access_ty
3828 ));
3829 }
3830 };
3831 let w_ty_ok = match widened_ty {
3832 I32 | I64 => true,
3833 _ => false,
3834 };
3835 assert!(w_ty_ok && widened_ty.bytes() >= access_ty.bytes());
3836
3837 assert!(expected_ty.bytes() >= access_ty.bytes());
3838 if expected_ty.bytes() > access_ty.bytes() {
3839 expected = builder.ins().ireduce(access_ty, expected);
3840 }
3841 assert!(replacement_ty.bytes() >= access_ty.bytes());
3842 if replacement_ty.bytes() > access_ty.bytes() {
3843 replacement = builder.ins().ireduce(access_ty, replacement);
3844 }
3845
3846 let (flags, _, addr) = unwrap_or_return_unreachable_state!(
3847 environ,
3848 prepare_atomic_addr(
3849 memarg,
3850 u8::try_from(access_ty.bytes()).unwrap(),
3851 builder,
3852 environ,
3853 )?
3854 );
3855 let mut res = builder.ins().atomic_cas(flags, addr, expected, replacement);
3856 if access_ty != widened_ty {
3857 res = builder.ins().uextend(widened_ty, res);
3858 }
3859 environ.stacks.push1(res);
3860 Ok(())
3861 }
3862
translate_atomic_load( widened_ty: Type, access_ty: Type, memarg: &MemArg, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>, ) -> WasmResult<()>3863 fn translate_atomic_load(
3864 widened_ty: Type,
3865 access_ty: Type,
3866 memarg: &MemArg,
3867 builder: &mut FunctionBuilder,
3868 environ: &mut FuncEnvironment<'_>,
3869 ) -> WasmResult<()> {
3870 // The load is performed at type `access_ty`, and the loaded value is zero extended
3871 // to `widened_ty`.
3872 match access_ty {
3873 I8 | I16 | I32 | I64 => {}
3874 _ => {
3875 return Err(wasm_unsupported!(
3876 "atomic_load: unsupported access type {:?}",
3877 access_ty
3878 ));
3879 }
3880 };
3881 let w_ty_ok = match widened_ty {
3882 I32 | I64 => true,
3883 _ => false,
3884 };
3885 assert!(w_ty_ok && widened_ty.bytes() >= access_ty.bytes());
3886
3887 let (flags, _, addr) = unwrap_or_return_unreachable_state!(
3888 environ,
3889 prepare_atomic_addr(
3890 memarg,
3891 u8::try_from(access_ty.bytes()).unwrap(),
3892 builder,
3893 environ,
3894 )?
3895 );
3896 let mut res = builder.ins().atomic_load(access_ty, flags, addr);
3897 if access_ty != widened_ty {
3898 res = builder.ins().uextend(widened_ty, res);
3899 }
3900 environ.stacks.push1(res);
3901 Ok(())
3902 }
3903
translate_atomic_store( access_ty: Type, memarg: &MemArg, builder: &mut FunctionBuilder, environ: &mut FuncEnvironment<'_>, ) -> WasmResult<()>3904 fn translate_atomic_store(
3905 access_ty: Type,
3906 memarg: &MemArg,
3907 builder: &mut FunctionBuilder,
3908 environ: &mut FuncEnvironment<'_>,
3909 ) -> WasmResult<()> {
3910 let mut data = environ.stacks.pop1();
3911 let data_ty = builder.func.dfg.value_type(data);
3912
3913 // The operation is performed at type `access_ty`, and the data to be stored may first
3914 // need to be narrowed accordingly.
3915 match access_ty {
3916 I8 | I16 | I32 | I64 => {}
3917 _ => {
3918 return Err(wasm_unsupported!(
3919 "atomic_store: unsupported access type {:?}",
3920 access_ty
3921 ));
3922 }
3923 };
3924 let d_ty_ok = match data_ty {
3925 I32 | I64 => true,
3926 _ => false,
3927 };
3928 assert!(d_ty_ok && data_ty.bytes() >= access_ty.bytes());
3929
3930 if data_ty.bytes() > access_ty.bytes() {
3931 data = builder.ins().ireduce(access_ty, data);
3932 }
3933
3934 let (flags, _, addr) = unwrap_or_return_unreachable_state!(
3935 environ,
3936 prepare_atomic_addr(
3937 memarg,
3938 u8::try_from(access_ty.bytes()).unwrap(),
3939 builder,
3940 environ,
3941 )?
3942 );
3943 builder.ins().atomic_store(flags, data, addr);
3944 Ok(())
3945 }
3946
translate_vector_icmp( cc: IntCC, needed_type: Type, builder: &mut FunctionBuilder, env: &mut FuncEnvironment<'_>, )3947 fn translate_vector_icmp(
3948 cc: IntCC,
3949 needed_type: Type,
3950 builder: &mut FunctionBuilder,
3951 env: &mut FuncEnvironment<'_>,
3952 ) {
3953 let (a, b) = env.stacks.pop2();
3954 let bitcast_a = optionally_bitcast_vector(a, needed_type, builder);
3955 let bitcast_b = optionally_bitcast_vector(b, needed_type, builder);
3956 env.stacks
3957 .push1(builder.ins().icmp(cc, bitcast_a, bitcast_b))
3958 }
3959
translate_fcmp(cc: FloatCC, builder: &mut FunctionBuilder, env: &mut FuncEnvironment<'_>)3960 fn translate_fcmp(cc: FloatCC, builder: &mut FunctionBuilder, env: &mut FuncEnvironment<'_>) {
3961 let (arg0, arg1) = env.stacks.pop2();
3962 let val = builder.ins().fcmp(cc, arg0, arg1);
3963 env.stacks.push1(builder.ins().uextend(I32, val));
3964 }
3965
translate_vector_fcmp( cc: FloatCC, needed_type: Type, builder: &mut FunctionBuilder, env: &mut FuncEnvironment<'_>, )3966 fn translate_vector_fcmp(
3967 cc: FloatCC,
3968 needed_type: Type,
3969 builder: &mut FunctionBuilder,
3970 env: &mut FuncEnvironment<'_>,
3971 ) {
3972 let (a, b) = env.stacks.pop2();
3973 let bitcast_a = optionally_bitcast_vector(a, needed_type, builder);
3974 let bitcast_b = optionally_bitcast_vector(b, needed_type, builder);
3975 env.stacks
3976 .push1(builder.ins().fcmp(cc, bitcast_a, bitcast_b))
3977 }
3978
translate_br_if( relative_depth: u32, builder: &mut FunctionBuilder, env: &mut FuncEnvironment<'_>, )3979 fn translate_br_if(
3980 relative_depth: u32,
3981 builder: &mut FunctionBuilder,
3982 env: &mut FuncEnvironment<'_>,
3983 ) {
3984 let val = env.stacks.pop1();
3985 let (br_destination, inputs) = translate_br_if_args(relative_depth, env);
3986 let next_block = builder.create_block();
3987 canonicalise_brif(builder, val, br_destination, inputs, next_block, &[]);
3988
3989 builder.seal_block(next_block); // The only predecessor is the current block.
3990 builder.switch_to_block(next_block);
3991 }
3992
translate_br_if_args<'a>( relative_depth: u32, env: &'a mut FuncEnvironment<'_>, ) -> (ir::Block, &'a mut [ir::Value])3993 fn translate_br_if_args<'a>(
3994 relative_depth: u32,
3995 env: &'a mut FuncEnvironment<'_>,
3996 ) -> (ir::Block, &'a mut [ir::Value]) {
3997 let i = env.stacks.control_stack.len() - 1 - (relative_depth as usize);
3998 let (return_count, br_destination) = {
3999 let frame = &mut env.stacks.control_stack[i];
4000 // The values returned by the branch are still available for the reachable
4001 // code that comes after it
4002 frame.set_branched_to_exit();
4003 let return_count = if frame.is_loop() {
4004 frame.num_param_values()
4005 } else {
4006 frame.num_return_values()
4007 };
4008 (return_count, frame.br_destination())
4009 };
4010 let inputs = env.stacks.peekn_mut(return_count);
4011 (br_destination, inputs)
4012 }
4013
4014 /// Determine the returned value type of a WebAssembly operator
type_of(operator: &Operator) -> Type4015 fn type_of(operator: &Operator) -> Type {
4016 match operator {
4017 Operator::V128Load { .. }
4018 | Operator::V128Store { .. }
4019 | Operator::V128Const { .. }
4020 | Operator::V128Not
4021 | Operator::V128And
4022 | Operator::V128AndNot
4023 | Operator::V128Or
4024 | Operator::V128Xor
4025 | Operator::V128AnyTrue
4026 | Operator::V128Bitselect => I8X16, // default type representing V128
4027
4028 Operator::I8x16Shuffle { .. }
4029 | Operator::I8x16Splat
4030 | Operator::V128Load8Splat { .. }
4031 | Operator::V128Load8Lane { .. }
4032 | Operator::V128Store8Lane { .. }
4033 | Operator::I8x16ExtractLaneS { .. }
4034 | Operator::I8x16ExtractLaneU { .. }
4035 | Operator::I8x16ReplaceLane { .. }
4036 | Operator::I8x16Eq
4037 | Operator::I8x16Ne
4038 | Operator::I8x16LtS
4039 | Operator::I8x16LtU
4040 | Operator::I8x16GtS
4041 | Operator::I8x16GtU
4042 | Operator::I8x16LeS
4043 | Operator::I8x16LeU
4044 | Operator::I8x16GeS
4045 | Operator::I8x16GeU
4046 | Operator::I8x16Neg
4047 | Operator::I8x16Abs
4048 | Operator::I8x16AllTrue
4049 | Operator::I8x16Shl
4050 | Operator::I8x16ShrS
4051 | Operator::I8x16ShrU
4052 | Operator::I8x16Add
4053 | Operator::I8x16AddSatS
4054 | Operator::I8x16AddSatU
4055 | Operator::I8x16Sub
4056 | Operator::I8x16SubSatS
4057 | Operator::I8x16SubSatU
4058 | Operator::I8x16MinS
4059 | Operator::I8x16MinU
4060 | Operator::I8x16MaxS
4061 | Operator::I8x16MaxU
4062 | Operator::I8x16AvgrU
4063 | Operator::I8x16Bitmask
4064 | Operator::I8x16Popcnt
4065 | Operator::I8x16RelaxedLaneselect => I8X16,
4066
4067 Operator::I16x8Splat
4068 | Operator::V128Load16Splat { .. }
4069 | Operator::V128Load16Lane { .. }
4070 | Operator::V128Store16Lane { .. }
4071 | Operator::I16x8ExtractLaneS { .. }
4072 | Operator::I16x8ExtractLaneU { .. }
4073 | Operator::I16x8ReplaceLane { .. }
4074 | Operator::I16x8Eq
4075 | Operator::I16x8Ne
4076 | Operator::I16x8LtS
4077 | Operator::I16x8LtU
4078 | Operator::I16x8GtS
4079 | Operator::I16x8GtU
4080 | Operator::I16x8LeS
4081 | Operator::I16x8LeU
4082 | Operator::I16x8GeS
4083 | Operator::I16x8GeU
4084 | Operator::I16x8Neg
4085 | Operator::I16x8Abs
4086 | Operator::I16x8AllTrue
4087 | Operator::I16x8Shl
4088 | Operator::I16x8ShrS
4089 | Operator::I16x8ShrU
4090 | Operator::I16x8Add
4091 | Operator::I16x8AddSatS
4092 | Operator::I16x8AddSatU
4093 | Operator::I16x8Sub
4094 | Operator::I16x8SubSatS
4095 | Operator::I16x8SubSatU
4096 | Operator::I16x8MinS
4097 | Operator::I16x8MinU
4098 | Operator::I16x8MaxS
4099 | Operator::I16x8MaxU
4100 | Operator::I16x8AvgrU
4101 | Operator::I16x8Mul
4102 | Operator::I16x8Bitmask
4103 | Operator::I16x8RelaxedLaneselect => I16X8,
4104
4105 Operator::I32x4Splat
4106 | Operator::V128Load32Splat { .. }
4107 | Operator::V128Load32Lane { .. }
4108 | Operator::V128Store32Lane { .. }
4109 | Operator::I32x4ExtractLane { .. }
4110 | Operator::I32x4ReplaceLane { .. }
4111 | Operator::I32x4Eq
4112 | Operator::I32x4Ne
4113 | Operator::I32x4LtS
4114 | Operator::I32x4LtU
4115 | Operator::I32x4GtS
4116 | Operator::I32x4GtU
4117 | Operator::I32x4LeS
4118 | Operator::I32x4LeU
4119 | Operator::I32x4GeS
4120 | Operator::I32x4GeU
4121 | Operator::I32x4Neg
4122 | Operator::I32x4Abs
4123 | Operator::I32x4AllTrue
4124 | Operator::I32x4Shl
4125 | Operator::I32x4ShrS
4126 | Operator::I32x4ShrU
4127 | Operator::I32x4Add
4128 | Operator::I32x4Sub
4129 | Operator::I32x4Mul
4130 | Operator::I32x4MinS
4131 | Operator::I32x4MinU
4132 | Operator::I32x4MaxS
4133 | Operator::I32x4MaxU
4134 | Operator::I32x4Bitmask
4135 | Operator::I32x4TruncSatF32x4S
4136 | Operator::I32x4TruncSatF32x4U
4137 | Operator::I32x4RelaxedLaneselect
4138 | Operator::V128Load32Zero { .. } => I32X4,
4139
4140 Operator::I64x2Splat
4141 | Operator::V128Load64Splat { .. }
4142 | Operator::V128Load64Lane { .. }
4143 | Operator::V128Store64Lane { .. }
4144 | Operator::I64x2ExtractLane { .. }
4145 | Operator::I64x2ReplaceLane { .. }
4146 | Operator::I64x2Eq
4147 | Operator::I64x2Ne
4148 | Operator::I64x2LtS
4149 | Operator::I64x2GtS
4150 | Operator::I64x2LeS
4151 | Operator::I64x2GeS
4152 | Operator::I64x2Neg
4153 | Operator::I64x2Abs
4154 | Operator::I64x2AllTrue
4155 | Operator::I64x2Shl
4156 | Operator::I64x2ShrS
4157 | Operator::I64x2ShrU
4158 | Operator::I64x2Add
4159 | Operator::I64x2Sub
4160 | Operator::I64x2Mul
4161 | Operator::I64x2Bitmask
4162 | Operator::I64x2RelaxedLaneselect
4163 | Operator::V128Load64Zero { .. } => I64X2,
4164
4165 Operator::F32x4Splat
4166 | Operator::F32x4ExtractLane { .. }
4167 | Operator::F32x4ReplaceLane { .. }
4168 | Operator::F32x4Eq
4169 | Operator::F32x4Ne
4170 | Operator::F32x4Lt
4171 | Operator::F32x4Gt
4172 | Operator::F32x4Le
4173 | Operator::F32x4Ge
4174 | Operator::F32x4Abs
4175 | Operator::F32x4Neg
4176 | Operator::F32x4Sqrt
4177 | Operator::F32x4Add
4178 | Operator::F32x4Sub
4179 | Operator::F32x4Mul
4180 | Operator::F32x4Div
4181 | Operator::F32x4Min
4182 | Operator::F32x4Max
4183 | Operator::F32x4PMin
4184 | Operator::F32x4PMax
4185 | Operator::F32x4ConvertI32x4S
4186 | Operator::F32x4ConvertI32x4U
4187 | Operator::F32x4Ceil
4188 | Operator::F32x4Floor
4189 | Operator::F32x4Trunc
4190 | Operator::F32x4Nearest
4191 | Operator::F32x4RelaxedMax
4192 | Operator::F32x4RelaxedMin
4193 | Operator::F32x4RelaxedMadd
4194 | Operator::F32x4RelaxedNmadd => F32X4,
4195
4196 Operator::F64x2Splat
4197 | Operator::F64x2ExtractLane { .. }
4198 | Operator::F64x2ReplaceLane { .. }
4199 | Operator::F64x2Eq
4200 | Operator::F64x2Ne
4201 | Operator::F64x2Lt
4202 | Operator::F64x2Gt
4203 | Operator::F64x2Le
4204 | Operator::F64x2Ge
4205 | Operator::F64x2Abs
4206 | Operator::F64x2Neg
4207 | Operator::F64x2Sqrt
4208 | Operator::F64x2Add
4209 | Operator::F64x2Sub
4210 | Operator::F64x2Mul
4211 | Operator::F64x2Div
4212 | Operator::F64x2Min
4213 | Operator::F64x2Max
4214 | Operator::F64x2PMin
4215 | Operator::F64x2PMax
4216 | Operator::F64x2Ceil
4217 | Operator::F64x2Floor
4218 | Operator::F64x2Trunc
4219 | Operator::F64x2Nearest
4220 | Operator::F64x2RelaxedMax
4221 | Operator::F64x2RelaxedMin
4222 | Operator::F64x2RelaxedMadd
4223 | Operator::F64x2RelaxedNmadd => F64X2,
4224
4225 _ => unimplemented!(
4226 "Currently only SIMD instructions are mapped to their return type; the \
4227 following instruction is not mapped: {:?}",
4228 operator
4229 ),
4230 }
4231 }
4232
4233 /// Some SIMD operations only operate on I8X16 in CLIF; this will convert them to that type by
4234 /// adding a bitcast if necessary.
optionally_bitcast_vector( value: Value, needed_type: Type, builder: &mut FunctionBuilder, ) -> Value4235 fn optionally_bitcast_vector(
4236 value: Value,
4237 needed_type: Type,
4238 builder: &mut FunctionBuilder,
4239 ) -> Value {
4240 if builder.func.dfg.value_type(value) != needed_type {
4241 let mut flags = MemFlags::new();
4242 flags.set_endianness(ir::Endianness::Little);
4243 builder.ins().bitcast(needed_type, flags, value)
4244 } else {
4245 value
4246 }
4247 }
4248
4249 #[inline(always)]
is_non_canonical_v128(ty: ir::Type) -> bool4250 fn is_non_canonical_v128(ty: ir::Type) -> bool {
4251 match ty {
4252 I64X2 | I32X4 | I16X8 | F32X4 | F64X2 => true,
4253 _ => false,
4254 }
4255 }
4256
4257 /// Cast to I8X16, any vector values in `values` that are of "non-canonical" type (meaning, not
4258 /// I8X16), and return them in a slice. A pre-scan is made to determine whether any casts are
4259 /// actually necessary, and if not, the original slice is returned. Otherwise the cast values
4260 /// are returned in a slice that belongs to the caller-supplied `SmallVec`.
canonicalise_v128_values<'a>( tmp_canonicalised: &'a mut SmallVec<[BlockArg; 16]>, builder: &mut FunctionBuilder, values: &'a [ir::Value], ) -> &'a [BlockArg]4261 fn canonicalise_v128_values<'a>(
4262 tmp_canonicalised: &'a mut SmallVec<[BlockArg; 16]>,
4263 builder: &mut FunctionBuilder,
4264 values: &'a [ir::Value],
4265 ) -> &'a [BlockArg] {
4266 debug_assert!(tmp_canonicalised.is_empty());
4267 // Cast, and push the resulting `Value`s into `canonicalised`.
4268 for v in values {
4269 let value = if is_non_canonical_v128(builder.func.dfg.value_type(*v)) {
4270 let mut flags = MemFlags::new();
4271 flags.set_endianness(ir::Endianness::Little);
4272 builder.ins().bitcast(I8X16, flags, *v)
4273 } else {
4274 *v
4275 };
4276 tmp_canonicalised.push(BlockArg::from(value));
4277 }
4278 tmp_canonicalised.as_slice()
4279 }
4280
4281 /// Generate a `jump` instruction, but first cast all 128-bit vector values to I8X16 if they
4282 /// don't have that type. This is done in somewhat roundabout way so as to ensure that we
4283 /// almost never have to do any heap allocation.
canonicalise_then_jump( builder: &mut FunctionBuilder, destination: ir::Block, params: &[ir::Value], ) -> ir::Inst4284 fn canonicalise_then_jump(
4285 builder: &mut FunctionBuilder,
4286 destination: ir::Block,
4287 params: &[ir::Value],
4288 ) -> ir::Inst {
4289 let mut tmp_canonicalised = SmallVec::<[_; 16]>::new();
4290 let canonicalised = canonicalise_v128_values(&mut tmp_canonicalised, builder, params);
4291 builder.ins().jump(destination, canonicalised)
4292 }
4293
4294 /// The same but for a `brif` instruction.
canonicalise_brif( builder: &mut FunctionBuilder, cond: ir::Value, block_then: ir::Block, params_then: &[ir::Value], block_else: ir::Block, params_else: &[ir::Value], ) -> ir::Inst4295 fn canonicalise_brif(
4296 builder: &mut FunctionBuilder,
4297 cond: ir::Value,
4298 block_then: ir::Block,
4299 params_then: &[ir::Value],
4300 block_else: ir::Block,
4301 params_else: &[ir::Value],
4302 ) -> ir::Inst {
4303 let mut tmp_canonicalised_then = SmallVec::<[_; 16]>::new();
4304 let canonicalised_then =
4305 canonicalise_v128_values(&mut tmp_canonicalised_then, builder, params_then);
4306 let mut tmp_canonicalised_else = SmallVec::<[_; 16]>::new();
4307 let canonicalised_else =
4308 canonicalise_v128_values(&mut tmp_canonicalised_else, builder, params_else);
4309 builder.ins().brif(
4310 cond,
4311 block_then,
4312 canonicalised_then,
4313 block_else,
4314 canonicalised_else,
4315 )
4316 }
4317
4318 /// A helper for popping and bitcasting a single value; since SIMD values can lose their type by
4319 /// using v128 (i.e. CLIF's I8x16) we must re-type the values using a bitcast to avoid CLIF
4320 /// typing issues.
pop1_with_bitcast( env: &mut FuncEnvironment<'_>, needed_type: Type, builder: &mut FunctionBuilder, ) -> Value4321 fn pop1_with_bitcast(
4322 env: &mut FuncEnvironment<'_>,
4323 needed_type: Type,
4324 builder: &mut FunctionBuilder,
4325 ) -> Value {
4326 optionally_bitcast_vector(env.stacks.pop1(), needed_type, builder)
4327 }
4328
4329 /// A helper for popping and bitcasting two values; since SIMD values can lose their type by
4330 /// using v128 (i.e. CLIF's I8x16) we must re-type the values using a bitcast to avoid CLIF
4331 /// typing issues.
pop2_with_bitcast( env: &mut FuncEnvironment<'_>, needed_type: Type, builder: &mut FunctionBuilder, ) -> (Value, Value)4332 fn pop2_with_bitcast(
4333 env: &mut FuncEnvironment<'_>,
4334 needed_type: Type,
4335 builder: &mut FunctionBuilder,
4336 ) -> (Value, Value) {
4337 let (a, b) = env.stacks.pop2();
4338 let bitcast_a = optionally_bitcast_vector(a, needed_type, builder);
4339 let bitcast_b = optionally_bitcast_vector(b, needed_type, builder);
4340 (bitcast_a, bitcast_b)
4341 }
4342
pop3_with_bitcast( env: &mut FuncEnvironment<'_>, needed_type: Type, builder: &mut FunctionBuilder, ) -> (Value, Value, Value)4343 fn pop3_with_bitcast(
4344 env: &mut FuncEnvironment<'_>,
4345 needed_type: Type,
4346 builder: &mut FunctionBuilder,
4347 ) -> (Value, Value, Value) {
4348 let (a, b, c) = env.stacks.pop3();
4349 let bitcast_a = optionally_bitcast_vector(a, needed_type, builder);
4350 let bitcast_b = optionally_bitcast_vector(b, needed_type, builder);
4351 let bitcast_c = optionally_bitcast_vector(c, needed_type, builder);
4352 (bitcast_a, bitcast_b, bitcast_c)
4353 }
4354
bitcast_arguments<'a>( builder: &FunctionBuilder, arguments: &'a mut [Value], params: &[ir::AbiParam], param_predicate: impl Fn(usize) -> bool, ) -> Vec<(Type, &'a mut Value)>4355 fn bitcast_arguments<'a>(
4356 builder: &FunctionBuilder,
4357 arguments: &'a mut [Value],
4358 params: &[ir::AbiParam],
4359 param_predicate: impl Fn(usize) -> bool,
4360 ) -> Vec<(Type, &'a mut Value)> {
4361 let filtered_param_types = params
4362 .iter()
4363 .enumerate()
4364 .filter(|(i, _)| param_predicate(*i))
4365 .map(|(_, param)| param.value_type);
4366
4367 // zip_eq, from the itertools::Itertools trait, is like Iterator::zip but panics if one
4368 // iterator ends before the other. The `param_predicate` is required to select exactly as many
4369 // elements of `params` as there are elements in `arguments`.
4370 let pairs = filtered_param_types.zip_eq(arguments.iter_mut());
4371
4372 // The arguments which need to be bitcasted are those which have some vector type but the type
4373 // expected by the parameter is not the same vector type as that of the provided argument.
4374 pairs
4375 .filter(|(param_type, _)| param_type.is_vector())
4376 .filter(|(param_type, arg)| {
4377 let arg_type = builder.func.dfg.value_type(**arg);
4378 assert!(
4379 arg_type.is_vector(),
4380 "unexpected type mismatch: expected {}, argument {} was actually of type {}",
4381 param_type,
4382 *arg,
4383 arg_type
4384 );
4385
4386 // This is the same check that would be done by `optionally_bitcast_vector`, except we
4387 // can't take a mutable borrow of the FunctionBuilder here, so we defer inserting the
4388 // bitcast instruction to the caller.
4389 arg_type != *param_type
4390 })
4391 .collect()
4392 }
4393
4394 /// A helper for bitcasting a sequence of return values for the function currently being built. If
4395 /// a value is a vector type that does not match its expected type, this will modify the value in
4396 /// place to point to the result of a `bitcast`. This conversion is necessary to translate Wasm
4397 /// code that uses `V128` as function parameters (or implicitly in block parameters) and still use
4398 /// specific CLIF types (e.g. `I32X4`) in the function body.
bitcast_wasm_returns(arguments: &mut [Value], builder: &mut FunctionBuilder)4399 pub fn bitcast_wasm_returns(arguments: &mut [Value], builder: &mut FunctionBuilder) {
4400 let changes = bitcast_arguments(builder, arguments, &builder.func.signature.returns, |i| {
4401 builder.func.signature.returns[i].purpose == ir::ArgumentPurpose::Normal
4402 });
4403 for (t, arg) in changes {
4404 let mut flags = MemFlags::new();
4405 flags.set_endianness(ir::Endianness::Little);
4406 *arg = builder.ins().bitcast(t, flags, *arg);
4407 }
4408 }
4409
4410 /// Like `bitcast_wasm_returns`, but for the parameters being passed to a specified callee.
bitcast_wasm_params( environ: &mut FuncEnvironment<'_>, callee_signature: ir::SigRef, arguments: &mut [Value], builder: &mut FunctionBuilder, )4411 fn bitcast_wasm_params(
4412 environ: &mut FuncEnvironment<'_>,
4413 callee_signature: ir::SigRef,
4414 arguments: &mut [Value],
4415 builder: &mut FunctionBuilder,
4416 ) {
4417 let callee_signature = &builder.func.dfg.signatures[callee_signature];
4418 let changes = bitcast_arguments(builder, arguments, &callee_signature.params, |i| {
4419 environ.is_wasm_parameter(i)
4420 });
4421 for (t, arg) in changes {
4422 let mut flags = MemFlags::new();
4423 flags.set_endianness(ir::Endianness::Little);
4424 *arg = builder.ins().bitcast(t, flags, *arg);
4425 }
4426 }
4427
create_catch_block( builder: &mut FunctionBuilder, catch: &wasmparser::Catch, environ: &mut FuncEnvironment<'_>, ) -> WasmResult<ir::Block>4428 fn create_catch_block(
4429 builder: &mut FunctionBuilder,
4430 catch: &wasmparser::Catch,
4431 environ: &mut FuncEnvironment<'_>,
4432 ) -> WasmResult<ir::Block> {
4433 let (is_ref, tag, label) = match catch {
4434 wasmparser::Catch::One { tag, label } => (false, Some(*tag), *label),
4435 wasmparser::Catch::OneRef { tag, label } => (true, Some(*tag), *label),
4436 wasmparser::Catch::All { label } => (false, None, *label),
4437 wasmparser::Catch::AllRef { label } => (true, None, *label),
4438 };
4439
4440 // We always create a handler block with one blockparam for the
4441 // one exception payload value that we use (`exn0` block-call
4442 // argument). This one payload value is the `exnref`. Note,
4443 // however, that we carry it in a native host-pointer-sized
4444 // payload (because this is what the exception ABI in Cranelift
4445 // requires). We then generate the args for the actual branch to
4446 // the handler block: we add unboxing code to load each value in
4447 // the exception signature if a specific tag is expected (hence
4448 // signature is known), and then append the `exnref` itself if we
4449 // are compiling a `*Ref` variant.
4450
4451 let (exn_ref_ty, needs_stack_map) = environ.reference_type(WasmHeapType::Exn);
4452 let (exn_payload_wasm_ty, exn_payload_ty) = match environ.pointer_type().bits() {
4453 32 => (wasmparser::ValType::I32, I32),
4454 64 => (wasmparser::ValType::I64, I64),
4455 _ => panic!("Unsupported pointer width"),
4456 };
4457 let block = block_with_params(builder, [exn_payload_wasm_ty], environ)?;
4458 builder.switch_to_block(block);
4459 let exn_ref = builder.func.dfg.block_params(block)[0];
4460 debug_assert!(exn_ref_ty.bits() <= exn_payload_ty.bits());
4461 let exn_ref = if exn_ref_ty.bits() < exn_payload_ty.bits() {
4462 builder.ins().ireduce(exn_ref_ty, exn_ref)
4463 } else {
4464 exn_ref
4465 };
4466
4467 if needs_stack_map {
4468 builder.declare_value_needs_stack_map(exn_ref);
4469 }
4470
4471 // We encode tag indices from the module directly as Cranelift
4472 // `ExceptionTag`s. We will translate those to (instance,
4473 // defined-tag-index) pairs during the unwind walk -- necessarily
4474 // dynamic because tag imports are provided only at instantiation
4475 // time.
4476 let clif_tag = tag.map(|t| ExceptionTag::from_u32(t));
4477
4478 environ.stacks.handlers.add_handler(clif_tag, block);
4479
4480 let mut params = vec![];
4481
4482 if let Some(tag) = tag {
4483 let tag = TagIndex::from_u32(tag);
4484 params.extend(environ.translate_exn_unbox(builder, tag, exn_ref)?);
4485 }
4486 if is_ref {
4487 params.push(exn_ref);
4488 }
4489
4490 // Generate the branch itself.
4491 let i = environ.stacks.control_stack.len() - 1 - (label as usize);
4492 let frame = &mut environ.stacks.control_stack[i];
4493 frame.set_branched_to_exit();
4494 canonicalise_then_jump(builder, frame.br_destination(), ¶ms);
4495
4496 Ok(block)
4497 }
4498