1 // A WORD OF CAUTION
2 //
3 // This entire file basically needs to be kept in sync with itself. It's not
4 // really possible to modify just one bit of this file without understanding
5 // all the other bits. Documentation tries to reference various bits here and
6 // there but try to make sure to read over everything before tweaking things!
7 //
8 // Also at this time this file is heavily based off the x86_64 file, so you'll
9 // probably want to read that one as well.
10 //
11 // Finally, control flow integrity hardening has been applied to the code using
12 // the Pointer Authentication (PAuth) and Branch Target Identification (BTI)
13 // technologies from the Arm instruction set architecture:
14 // * All callable functions start with either the `BTI c` or `PACIASP`/`PACIBSP`
15 //   instructions
16 // * Return addresses are signed and authenticated using the stack pointer
17 //   value as a modifier (similarly to the salt in a HMAC operation); the
18 //   `DW_CFA_AARCH64_negate_ra_state` DWARF operation (aliased with the
19 //   `.cfi_window_save` assembler directive) informs an unwinder about this
20 
21 use super::wasmtime_fiber_start;
22 use wasmtime_asm_macros::asm_func;
23 
24 cfg_if::cfg_if! {
25     if #[cfg(target_vendor = "apple")] {
26         macro_rules! paci1716 { () => ("pacib1716\n"); }
27         macro_rules! pacisp { () => ("pacibsp\n"); }
28         macro_rules! autisp { () => ("autibsp\n"); }
29         macro_rules! sym_adrp { ($s:tt) => (concat!($s, "@PAGE")); }
30         macro_rules! sym_add { ($s:tt) => (concat!($s, "@PAGEOFF")); }
31     } else {
32         macro_rules! paci1716 { () => ("pacia1716\n"); }
33         macro_rules! pacisp { () => ("paciasp\n"); }
34         macro_rules! autisp { () => ("autiasp\n"); }
35         macro_rules! sym_adrp { ($s:tt) => (concat!($s, "")); }
36         macro_rules! sym_add { ($s:tt) => (concat!(":lo12:", $s)); }
37     }
38 }
39 
40 // fn(top_of_stack(%x0): *mut u8)
41 asm_func!(
42     wasmtime_versioned_export_macros::versioned_stringify_ident!(wasmtime_fiber_switch),
43     concat!(
44         "
45             .cfi_startproc
46         ",
47         pacisp!(),
48         "
49             .cfi_window_save
50             // Save all callee-saved registers on the stack since we're
51             // assuming they're clobbered as a result of the stack switch.
52             stp x29, x30, [sp, -16]!
53             stp x20, x19, [sp, -16]!
54             stp x22, x21, [sp, -16]!
55             stp x24, x23, [sp, -16]!
56             stp x26, x25, [sp, -16]!
57             stp x28, x27, [sp, -16]!
58             stp d9, d8, [sp, -16]!
59             stp d11, d10, [sp, -16]!
60             stp d13, d12, [sp, -16]!
61             stp d15, d14, [sp, -16]!
62 
63             // Load our previously saved stack pointer to resume to, and save
64             // off our current stack pointer on where to come back to
65             // eventually.
66             ldr x8, [x0, -0x10]
67             mov x9, sp
68             str x9, [x0, -0x10]
69 
70             // Switch to the new stack and restore all our callee-saved
71             // registers after the switch and return to our new stack.
72             mov sp, x8
73             ldp d15, d14, [sp], 16
74             ldp d13, d12, [sp], 16
75             ldp d11, d10, [sp], 16
76             ldp d9, d8, [sp], 16
77             ldp x28, x27, [sp], 16
78             ldp x26, x25, [sp], 16
79             ldp x24, x23, [sp], 16
80             ldp x22, x21, [sp], 16
81             ldp x20, x19, [sp], 16
82             ldp x29, x30, [sp], 16
83         ",
84         autisp!(),
85         "
86             .cfi_window_save
87             ret
88             .cfi_endproc
89         ",
90     ),
91 );
92 
93 // fn(
94 //    top_of_stack(%x0): *mut u8,
95 //    entry_point(%x1): extern fn(*mut u8, *mut u8),
96 //    entry_arg0(%x2): *mut u8,
97 // )
98 // We set up the newly initialized fiber, so that it resumes execution
99 // from wasmtime_fiber_start(). As a result, we need a signed address
100 // of this function, so there are 2 requirements:
101 // * The fiber stack pointer value that is used by the signing operation
102 //   must match the value when the pointer is authenticated inside
103 //   wasmtime_fiber_switch(), otherwise the latter would fault
104 // * We would like to use an instruction that is executed as a no-op by
105 //   processors that do not support PAuth, so that the code is
106 //   backward-compatible and there is no duplication; `PACIA1716` is a
107 //   suitable one, which has the following operand register
108 //   conventions:
109 //   * X17 contains the pointer value to sign
110 //   * X16 contains the modifier value
111 //
112 // TODO: Use the PACGA instruction to authenticate the saved register
113 // state, which avoids creating signed pointers to
114 // wasmtime_fiber_start(), and provides wider coverage.
115 #[rustfmt::skip]
116 asm_func!(
117     wasmtime_versioned_export_macros::versioned_stringify_ident!(wasmtime_fiber_init),
118     concat!(
119         "
120             .cfi_startproc
121             hint #34 // bti c
122             sub x16, x0, #16
123             adrp x17, ", sym_adrp!("{fiber}"), "
124             add x17, x17, ", sym_add!("{fiber}"), "
125         ",
126         paci1716!(),
127         "
128             str x17, [x16, -0x8] // x17 => lr
129             str x0, [x16, -0x18] // x0 => x19
130             stp x2, x1, [x0, -0x38] // x1 => x20, x2 => x21
131 
132             // `wasmtime_fiber_switch` has an 0xa0 byte stack, and we add 0x10 more for
133             // the original reserved 16 bytes.
134             add x8, x0, -0xb0
135             str x8, [x0, -0x10]
136             ret
137             .cfi_endproc
138         ",
139     ),
140     fiber = sym wasmtime_fiber_start,
141 );
142 
143 // See the x86_64 file for more commentary on what these CFI directives are
144 // doing. Like over there note that the relative offsets to registers here
145 // match the frame layout in `wasmtime_fiber_switch`.
146 asm_func!(
147     wasmtime_versioned_export_macros::versioned_stringify_ident!(wasmtime_fiber_start),
148     "
149         .cfi_startproc simple
150         .cfi_def_cfa_offset 0
151         .cfi_escape 0x0f,    /* DW_CFA_def_cfa_expression */ \
152             5,               /* the byte length of this expression */ \
153             0x6f,            /* DW_OP_reg31(%sp) */ \
154             0x06,            /* DW_OP_deref */ \
155             0x23, 0xa0, 0x1  /* DW_OP_plus_uconst 0xa0 */
156         .cfi_rel_offset x29, -0x10
157         .cfi_rel_offset x30, -0x08
158         .cfi_window_save
159         .cfi_rel_offset x19, -0x18
160         .cfi_rel_offset x20, -0x20
161         .cfi_rel_offset x21, -0x28
162         .cfi_rel_offset x22, -0x30
163         .cfi_rel_offset x23, -0x38
164         .cfi_rel_offset x24, -0x40
165         .cfi_rel_offset x25, -0x48
166         .cfi_rel_offset x26, -0x50
167         .cfi_rel_offset x27, -0x58
168 
169         // Load our two arguments from the stack, where x1 is our start
170         // procedure and x0 is its first argument. This also blows away the
171         // stack space used by those two arguments.
172         mov x0, x21
173         mov x1, x19
174 
175         // ... and then we call the function! Note that this is a function call
176         // so our frame stays on the stack to backtrace through.
177         blr x20
178         // Unreachable, here for safety. This should help catch unexpected
179         // behaviors.  Use a noticeable payload so one can grep for it in the
180         // codebase.
181         brk 0xf1b3
182         .cfi_endproc
183     ",
184 );
185