1 // A WORD OF CAUTION 2 // 3 // This entire file basically needs to be kept in sync with itself. It's not 4 // really possible to modify just one bit of this file without understanding 5 // all the other bits. Documentation tries to reference various bits here and 6 // there but try to make sure to read over everything before tweaking things! 7 // 8 // Also at this time this file is heavily based off the x86_64 file, so you'll 9 // probably want to read that one as well. 10 // 11 // Finally, control flow integrity hardening has been applied to the code using 12 // the Pointer Authentication (PAuth) and Branch Target Identification (BTI) 13 // technologies from the Arm instruction set architecture: 14 // * All callable functions start with either the `BTI c` or `PACIASP`/`PACIBSP` 15 // instructions 16 // * Return addresses are signed and authenticated using the stack pointer 17 // value as a modifier (similarly to the salt in a HMAC operation); the 18 // `DW_CFA_AARCH64_negate_ra_state` DWARF operation (aliased with the 19 // `.cfi_window_save` assembler directive) informs an unwinder about this 20 21 use core::arch::naked_asm; 22 23 cfg_if::cfg_if! { 24 if #[cfg(target_vendor = "apple")] { 25 macro_rules! paci1716 { () => ("pacib1716\n"); } 26 macro_rules! pacisp { () => ("pacibsp\n"); } 27 macro_rules! autisp { () => ("autibsp\n"); } 28 } else { 29 macro_rules! paci1716 { () => ("pacia1716\n"); } 30 macro_rules! pacisp { () => ("paciasp\n"); } 31 macro_rules! autisp { () => ("autiasp\n"); } 32 } 33 } 34 35 #[inline(never)] // FIXME(rust-lang/rust#148307) 36 pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) { 37 unsafe { wasmtime_fiber_switch_(top_of_stack) } 38 } 39 40 #[unsafe(naked)] 41 unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* x0 */) { 42 naked_asm!(concat!( 43 " 44 .cfi_startproc 45 ", 46 pacisp!(), 47 " 48 .cfi_window_save 49 // Save all callee-saved registers on the stack since we're 50 // assuming they're clobbered as a result of the stack switch. 51 stp x29, x30, [sp, -16]! 52 stp x27, x28, [sp, -16]! 53 stp x25, x26, [sp, -16]! 54 stp x23, x24, [sp, -16]! 55 stp x21, x22, [sp, -16]! 56 stp x19, x20, [sp, -16]! 57 stp d14, d15, [sp, -16]! 58 stp d12, d13, [sp, -16]! 59 stp d10, d11, [sp, -16]! 60 stp d8, d9, [sp, -16]! 61 62 // Load our previously saved stack pointer to resume to, and save 63 // off our current stack pointer on where to come back to 64 // eventually. 65 ldr x8, [x0, -0x10] 66 mov x9, sp 67 str x9, [x0, -0x10] 68 69 // Switch to the new stack and restore all our callee-saved 70 // registers after the switch and return to our new stack. 71 mov sp, x8 72 ldp d8, d9, [sp], 16 73 ldp d10, d11, [sp], 16 74 ldp d12, d13, [sp], 16 75 ldp d14, d15, [sp], 16 76 77 ldp x19, x20, [sp], 16 78 ldp x21, x22, [sp], 16 79 ldp x23, x24, [sp], 16 80 ldp x25, x26, [sp], 16 81 ldp x27, x28, [sp], 16 82 ldp x29, x30, [sp], 16 83 ", 84 autisp!(), 85 " 86 .cfi_window_save 87 ret 88 .cfi_endproc 89 ", 90 )); 91 } 92 93 pub(crate) unsafe fn wasmtime_fiber_init( 94 top_of_stack: *mut u8, 95 entry_point: extern "C" fn(*mut u8, *mut u8) -> *mut u8, 96 entry_arg0: *mut u8, // x2 97 ) { 98 #[repr(C)] 99 #[derive(Default)] 100 struct InitialStack { 101 d8: u64, 102 d9: u64, 103 d10: u64, 104 d11: u64, 105 d12: u64, 106 d13: u64, 107 d14: u64, 108 d15: u64, 109 110 x19: *mut u8, 111 x20: *mut u8, 112 x21: *mut u8, 113 x22: *mut u8, 114 x23: *mut u8, 115 x24: *mut u8, 116 x25: *mut u8, 117 x26: *mut u8, 118 x27: *mut u8, 119 x28: *mut u8, 120 121 fp: *mut u8, 122 lr: *mut u8, 123 124 // unix.rs reserved space 125 last_sp: *mut u8, 126 run_result: *mut u8, 127 } 128 129 unsafe { 130 let initial_stack = top_of_stack.cast::<InitialStack>().sub(1); 131 initial_stack.write(InitialStack { 132 x19: top_of_stack, 133 x20: entry_point as *mut u8, 134 x21: entry_arg0, 135 x22: wasmtime_fiber_switch_ as *mut u8, 136 137 // We set up the newly initialized fiber, so that it resumes 138 // execution from wasmtime_fiber_start(). As a result, we need a 139 // signed address of this function because `wasmtime_fiber_switch` 140 // ends with a `auti{a,b}sp` instruction. There are 2 requirements: 141 // * We would like to use an instruction that is executed as a no-op 142 // by processors that do not support PAuth, so that the code is 143 // backward-compatible and there is no duplication; `PACIA1716` is 144 // a suitable one. 145 // * The fiber stack pointer value that is used by the signing 146 // operation must match the value when the pointer is 147 // authenticated inside wasmtime_fiber_switch(), which is 16 bytes 148 // below the `top_of_stack` which will be `sp` at the time of the 149 // `auti{a,b}sp`. 150 // 151 // TODO: Use the PACGA instruction to authenticate the saved register 152 // state, which avoids creating signed pointers to 153 // wasmtime_fiber_start(), and provides wider coverage. 154 lr: paci1716(wasmtime_fiber_start as *mut u8, top_of_stack.sub(16)), 155 156 last_sp: initial_stack.cast(), 157 ..InitialStack::default() 158 }); 159 } 160 } 161 162 /// Signs `r17` with the value in `r16` using either `paci{a,b}1716` depending 163 /// on the platform. 164 fn paci1716(mut r17: *mut u8, r16: *mut u8) -> *mut u8 { 165 unsafe { 166 core::arch::asm!( 167 paci1716!(), 168 inout("x17") r17, 169 in("x16") r16, 170 ); 171 r17 172 } 173 } 174 175 // See the x86_64 file for more commentary on what these CFI directives are 176 // doing. Like over there note that the relative offsets to registers here 177 // match the frame layout in `wasmtime_fiber_switch`. 178 #[unsafe(naked)] 179 unsafe extern "C" fn wasmtime_fiber_start() -> ! { 180 naked_asm!( 181 " 182 .cfi_startproc simple 183 .cfi_def_cfa_offset 0 184 .cfi_escape 0x0f, /* DW_CFA_def_cfa_expression */ \ 185 5, /* the byte length of this expression */ \ 186 0x6f, /* DW_OP_reg31(%sp) */ \ 187 0x06, /* DW_OP_deref */ \ 188 0x23, 0xa0, 0x1 /* DW_OP_plus_uconst 0xa0 */ 189 .cfi_rel_offset x30, -0x08 190 .cfi_rel_offset x29, -0x10 191 .cfi_window_save 192 .cfi_rel_offset x28, -0x18 193 .cfi_rel_offset x27, -0x20 194 .cfi_rel_offset x26, -0x28 195 .cfi_rel_offset x25, -0x30 196 .cfi_rel_offset x24, -0x38 197 .cfi_rel_offset x23, -0x40 198 .cfi_rel_offset x22, -0x48 199 .cfi_rel_offset x21, -0x50 200 .cfi_rel_offset x20, -0x58 201 .cfi_rel_offset x19, -0x60 202 203 // Load our two arguments from the stack, where x1 is our start 204 // procedure and x0 is its first argument. This also blows away the 205 // stack space used by those two arguments. 206 mov x0, x21 207 mov x1, x19 208 209 // ... and then we call the function! Note that this is a function call 210 // so our frame stays on the stack to backtrace through. 211 blr x20 212 213 // The entry function returns where to switch to as the final switch, so 214 // that's performed here in inline assembly. 215 blr x22 216 217 // Unreachable, here for safety. This should help catch unexpected 218 // behaviors. Use a noticeable payload so one can grep for it in the 219 // codebase. 220 brk 0xf1b3 221 .cfi_endproc 222 ", 223 ); 224 } 225