1 // A WORD OF CAUTION 2 // 3 // This entire file basically needs to be kept in sync with itself. It's not 4 // really possible to modify just one bit of this file without understanding 5 // all the other bits. Documentation tries to reference various bits here and 6 // there but try to make sure to read over everything before tweaking things! 7 // 8 // Also at this time this file is heavily based off the x86_64 file, so you'll 9 // probably want to read that one as well. 10 // 11 // Finally, control flow integrity hardening has been applied to the code using 12 // the Pointer Authentication (PAuth) and Branch Target Identification (BTI) 13 // technologies from the Arm instruction set architecture: 14 // * All callable functions start with either the `BTI c` or `PACIASP`/`PACIBSP` 15 // instructions 16 // * Return addresses are signed and authenticated using the stack pointer 17 // value as a modifier (similarly to the salt in a HMAC operation); the 18 // `DW_CFA_AARCH64_negate_ra_state` DWARF operation (aliased with the 19 // `.cfi_window_save` assembler directive) informs an unwinder about this 20 21 use core::arch::naked_asm; 22 23 cfg_if::cfg_if! { 24 if #[cfg(target_vendor = "apple")] { 25 macro_rules! paci1716 { () => ("pacib1716\n"); } 26 macro_rules! pacisp { () => ("pacibsp\n"); } 27 macro_rules! autisp { () => ("autibsp\n"); } 28 } else { 29 macro_rules! paci1716 { () => ("pacia1716\n"); } 30 macro_rules! pacisp { () => ("paciasp\n"); } 31 macro_rules! autisp { () => ("autiasp\n"); } 32 } 33 } 34 35 #[inline(never)] // FIXME(rust-lang/rust#148307) 36 pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) { 37 unsafe { wasmtime_fiber_switch_(top_of_stack) } 38 } 39 40 #[unsafe(naked)] 41 unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* x0 */) { 42 naked_asm!(concat!( 43 " 44 .cfi_startproc 45 ", 46 pacisp!(), 47 " 48 .cfi_window_save 49 // Save all callee-saved registers on the stack since we're 50 // assuming they're clobbered as a result of the stack switch. 51 stp x29, x30, [sp, -16]! 52 stp x27, x28, [sp, -16]! 53 stp x25, x26, [sp, -16]! 54 stp x23, x24, [sp, -16]! 55 stp x21, x22, [sp, -16]! 56 stp x19, x20, [sp, -16]! 57 stp d14, d15, [sp, -16]! 58 stp d12, d13, [sp, -16]! 59 stp d10, d11, [sp, -16]! 60 stp d8, d9, [sp, -16]! 61 62 // Load our previously saved stack pointer to resume to, and save 63 // off our current stack pointer on where to come back to 64 // eventually. 65 ldr x8, [x0, -0x10] 66 mov x9, sp 67 str x9, [x0, -0x10] 68 69 // Switch to the new stack and restore all our callee-saved 70 // registers after the switch and return to our new stack. 71 mov sp, x8 72 ldp d8, d9, [sp], 16 73 ldp d10, d11, [sp], 16 74 ldp d12, d13, [sp], 16 75 ldp d14, d15, [sp], 16 76 77 ldp x19, x20, [sp], 16 78 ldp x21, x22, [sp], 16 79 ldp x23, x24, [sp], 16 80 ldp x25, x26, [sp], 16 81 ldp x27, x28, [sp], 16 82 ldp x29, x30, [sp], 16 83 ", 84 autisp!(), 85 " 86 .cfi_window_save 87 ret 88 .cfi_endproc 89 ", 90 )); 91 } 92 93 pub(crate) unsafe fn wasmtime_fiber_init( 94 top_of_stack: *mut u8, 95 entry_point: extern "C" fn(*mut u8, *mut u8), 96 entry_arg0: *mut u8, // x2 97 ) { 98 #[repr(C)] 99 #[derive(Default)] 100 struct InitialStack { 101 d8: u64, 102 d9: u64, 103 d10: u64, 104 d11: u64, 105 d12: u64, 106 d13: u64, 107 d14: u64, 108 d15: u64, 109 110 x19: *mut u8, 111 x20: *mut u8, 112 x21: *mut u8, 113 x22: *mut u8, 114 x23: *mut u8, 115 x24: *mut u8, 116 x25: *mut u8, 117 x26: *mut u8, 118 x27: *mut u8, 119 x28: *mut u8, 120 121 fp: *mut u8, 122 lr: *mut u8, 123 124 // unix.rs reserved space 125 last_sp: *mut u8, 126 run_result: *mut u8, 127 } 128 129 unsafe { 130 let initial_stack = top_of_stack.cast::<InitialStack>().sub(1); 131 initial_stack.write(InitialStack { 132 x19: top_of_stack, 133 x20: entry_point as *mut u8, 134 x21: entry_arg0, 135 136 // We set up the newly initialized fiber, so that it resumes 137 // execution from wasmtime_fiber_start(). As a result, we need a 138 // signed address of this function because `wasmtime_fiber_switch` 139 // ends with a `auti{a,b}sp` instruction. There are 2 requirements: 140 // * We would like to use an instruction that is executed as a no-op 141 // by processors that do not support PAuth, so that the code is 142 // backward-compatible and there is no duplication; `PACIA1716` is 143 // a suitable one. 144 // * The fiber stack pointer value that is used by the signing 145 // operation must match the value when the pointer is 146 // authenticated inside wasmtime_fiber_switch(), which is 16 bytes 147 // below the `top_of_stack` which will be `sp` at the time of the 148 // `auti{a,b}sp`. 149 // 150 // TODO: Use the PACGA instruction to authenticate the saved register 151 // state, which avoids creating signed pointers to 152 // wasmtime_fiber_start(), and provides wider coverage. 153 lr: paci1716(wasmtime_fiber_start as *mut u8, top_of_stack.sub(16)), 154 155 last_sp: initial_stack.cast(), 156 ..InitialStack::default() 157 }); 158 } 159 } 160 161 /// Signs `r17` with the value in `r16` using either `paci{a,b}1716` depending 162 /// on the platform. 163 fn paci1716(mut r17: *mut u8, r16: *mut u8) -> *mut u8 { 164 unsafe { 165 core::arch::asm!( 166 paci1716!(), 167 inout("x17") r17, 168 in("x16") r16, 169 ); 170 r17 171 } 172 } 173 174 // See the x86_64 file for more commentary on what these CFI directives are 175 // doing. Like over there note that the relative offsets to registers here 176 // match the frame layout in `wasmtime_fiber_switch`. 177 #[unsafe(naked)] 178 unsafe extern "C" fn wasmtime_fiber_start() -> ! { 179 naked_asm!( 180 " 181 .cfi_startproc simple 182 .cfi_def_cfa_offset 0 183 .cfi_escape 0x0f, /* DW_CFA_def_cfa_expression */ \ 184 5, /* the byte length of this expression */ \ 185 0x6f, /* DW_OP_reg31(%sp) */ \ 186 0x06, /* DW_OP_deref */ \ 187 0x23, 0xa0, 0x1 /* DW_OP_plus_uconst 0xa0 */ 188 .cfi_rel_offset x30, -0x08 189 .cfi_rel_offset x29, -0x10 190 .cfi_window_save 191 .cfi_rel_offset x28, -0x18 192 .cfi_rel_offset x27, -0x20 193 .cfi_rel_offset x26, -0x28 194 .cfi_rel_offset x25, -0x30 195 .cfi_rel_offset x24, -0x38 196 .cfi_rel_offset x23, -0x40 197 .cfi_rel_offset x22, -0x48 198 .cfi_rel_offset x21, -0x50 199 .cfi_rel_offset x20, -0x58 200 .cfi_rel_offset x19, -0x60 201 202 // Load our two arguments from the stack, where x1 is our start 203 // procedure and x0 is its first argument. This also blows away the 204 // stack space used by those two arguments. 205 mov x0, x21 206 mov x1, x19 207 208 // ... and then we call the function! Note that this is a function call 209 // so our frame stays on the stack to backtrace through. 210 blr x20 211 // Unreachable, here for safety. This should help catch unexpected 212 // behaviors. Use a noticeable payload so one can grep for it in the 213 // codebase. 214 brk 0xf1b3 215 .cfi_endproc 216 ", 217 ); 218 } 219