1 // A WORD OF CAUTION 2 // 3 // This entire file basically needs to be kept in sync with itself. It's not 4 // really possible to modify just one bit of this file without understanding 5 // all the other bits. Documentation tries to reference various bits here and 6 // there but try to make sure to read over everything before tweaking things! 7 // 8 // Also at this time this file is heavily based off the x86_64 file, so you'll 9 // probably want to read that one as well. 10 // 11 // Finally, control flow integrity hardening has been applied to the code using 12 // the Pointer Authentication (PAuth) and Branch Target Identification (BTI) 13 // technologies from the Arm instruction set architecture: 14 // * All callable functions start with either the `BTI c` or `PACIASP`/`PACIBSP` 15 // instructions 16 // * Return addresses are signed and authenticated using the stack pointer 17 // value as a modifier (similarly to the salt in a HMAC operation); the 18 // `DW_CFA_AARCH64_negate_ra_state` DWARF operation (aliased with the 19 // `.cfi_window_save` assembler directive) informs an unwinder about this 20 21 use core::arch::naked_asm; 22 23 cfg_if::cfg_if! { 24 if #[cfg(target_vendor = "apple")] { 25 macro_rules! paci1716 { () => ("pacib1716\n"); } 26 macro_rules! pacisp { () => ("pacibsp\n"); } 27 macro_rules! autisp { () => ("autibsp\n"); } 28 macro_rules! sym_adrp { ($s:tt) => (concat!($s, "@PAGE")); } 29 macro_rules! sym_add { ($s:tt) => (concat!($s, "@PAGEOFF")); } 30 } else { 31 macro_rules! paci1716 { () => ("pacia1716\n"); } 32 macro_rules! pacisp { () => ("paciasp\n"); } 33 macro_rules! autisp { () => ("autiasp\n"); } 34 macro_rules! sym_adrp { ($s:tt) => (concat!($s, "")); } 35 macro_rules! sym_add { ($s:tt) => (concat!(":lo12:", $s)); } 36 } 37 } 38 39 #[unsafe(naked)] 40 pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* x0 */) { 41 naked_asm!(concat!( 42 " 43 .cfi_startproc 44 ", 45 pacisp!(), 46 " 47 .cfi_window_save 48 // Save all callee-saved registers on the stack since we're 49 // assuming they're clobbered as a result of the stack switch. 50 stp x29, x30, [sp, -16]! 51 stp x20, x19, [sp, -16]! 52 stp x22, x21, [sp, -16]! 53 stp x24, x23, [sp, -16]! 54 stp x26, x25, [sp, -16]! 55 stp x28, x27, [sp, -16]! 56 stp d9, d8, [sp, -16]! 57 stp d11, d10, [sp, -16]! 58 stp d13, d12, [sp, -16]! 59 stp d15, d14, [sp, -16]! 60 61 // Load our previously saved stack pointer to resume to, and save 62 // off our current stack pointer on where to come back to 63 // eventually. 64 ldr x8, [x0, -0x10] 65 mov x9, sp 66 str x9, [x0, -0x10] 67 68 // Switch to the new stack and restore all our callee-saved 69 // registers after the switch and return to our new stack. 70 mov sp, x8 71 ldp d15, d14, [sp], 16 72 ldp d13, d12, [sp], 16 73 ldp d11, d10, [sp], 16 74 ldp d9, d8, [sp], 16 75 ldp x28, x27, [sp], 16 76 ldp x26, x25, [sp], 16 77 ldp x24, x23, [sp], 16 78 ldp x22, x21, [sp], 16 79 ldp x20, x19, [sp], 16 80 ldp x29, x30, [sp], 16 81 ", 82 autisp!(), 83 " 84 .cfi_window_save 85 ret 86 .cfi_endproc 87 ", 88 )); 89 } 90 91 // We set up the newly initialized fiber, so that it resumes execution 92 // from wasmtime_fiber_start(). As a result, we need a signed address 93 // of this function, so there are 2 requirements: 94 // * The fiber stack pointer value that is used by the signing operation 95 // must match the value when the pointer is authenticated inside 96 // wasmtime_fiber_switch(), otherwise the latter would fault 97 // * We would like to use an instruction that is executed as a no-op by 98 // processors that do not support PAuth, so that the code is 99 // backward-compatible and there is no duplication; `PACIA1716` is a 100 // suitable one, which has the following operand register 101 // conventions: 102 // * X17 contains the pointer value to sign 103 // * X16 contains the modifier value 104 // 105 // TODO: Use the PACGA instruction to authenticate the saved register 106 // state, which avoids creating signed pointers to 107 // wasmtime_fiber_start(), and provides wider coverage. 108 #[unsafe(naked)] 109 pub(crate) unsafe extern "C" fn wasmtime_fiber_init( 110 top_of_stack: *mut u8, // x0 111 entry_point: extern "C" fn(*mut u8, *mut u8), // x1 112 entry_arg0: *mut u8, // x2 113 ) { 114 naked_asm!( 115 concat!( 116 " 117 .cfi_startproc 118 hint #34 // bti c 119 sub x16, x0, #16 120 adrp x17, ", sym_adrp!("{fiber}"), " 121 add x17, x17, ", sym_add!("{fiber}"), " 122 ", 123 paci1716!(), 124 " 125 str x17, [x16, -0x8] // x17 => lr 126 str x0, [x16, -0x18] // x0 => x19 127 stp x2, x1, [x0, -0x38] // x1 => x20, x2 => x21 128 129 // `wasmtime_fiber_switch` has an 0xa0 byte stack, and we add 0x10 more for 130 // the original reserved 16 bytes. 131 add x8, x0, -0xb0 132 str x8, [x0, -0x10] 133 ret 134 .cfi_endproc 135 ", 136 ), 137 fiber = sym wasmtime_fiber_start, 138 ); 139 } 140 141 // See the x86_64 file for more commentary on what these CFI directives are 142 // doing. Like over there note that the relative offsets to registers here 143 // match the frame layout in `wasmtime_fiber_switch`. 144 #[unsafe(naked)] 145 unsafe extern "C" fn wasmtime_fiber_start() -> ! { 146 naked_asm!( 147 " 148 .cfi_startproc simple 149 .cfi_def_cfa_offset 0 150 .cfi_escape 0x0f, /* DW_CFA_def_cfa_expression */ \ 151 5, /* the byte length of this expression */ \ 152 0x6f, /* DW_OP_reg31(%sp) */ \ 153 0x06, /* DW_OP_deref */ \ 154 0x23, 0xa0, 0x1 /* DW_OP_plus_uconst 0xa0 */ 155 .cfi_rel_offset x29, -0x10 156 .cfi_rel_offset x30, -0x08 157 .cfi_window_save 158 .cfi_rel_offset x19, -0x18 159 .cfi_rel_offset x20, -0x20 160 .cfi_rel_offset x21, -0x28 161 .cfi_rel_offset x22, -0x30 162 .cfi_rel_offset x23, -0x38 163 .cfi_rel_offset x24, -0x40 164 .cfi_rel_offset x25, -0x48 165 .cfi_rel_offset x26, -0x50 166 .cfi_rel_offset x27, -0x58 167 168 // Load our two arguments from the stack, where x1 is our start 169 // procedure and x0 is its first argument. This also blows away the 170 // stack space used by those two arguments. 171 mov x0, x21 172 mov x1, x19 173 174 // ... and then we call the function! Note that this is a function call 175 // so our frame stays on the stack to backtrace through. 176 blr x20 177 // Unreachable, here for safety. This should help catch unexpected 178 // behaviors. Use a noticeable payload so one can grep for it in the 179 // codebase. 180 brk 0xf1b3 181 .cfi_endproc 182 ", 183 ); 184 } 185