1/*- 2 * Copyright (c) 2014 Andrew Turner 3 * Copyright (c) 2014-2015 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner 7 * under sponsorship from the FreeBSD Foundation 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include <machine/asm.h> 33__FBSDID("$FreeBSD$"); 34 35#include <machine/setjmp.h> 36#include <machine/param.h> 37#include <machine/vmparam.h> 38 39#include "assym.inc" 40 41.macro check_user_access user_arg, limit, bad_addr_func 42 ldr x7, =(\limit) 43 cmp x\user_arg, x7 44 b.cs \bad_addr_func 45.endm 46 47/* 48 * One of the fu* or su* functions failed, return -1. 49 */ 50ENTRY(fsu_fault) 51 SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */ 52 EXIT_USER_ACCESS_CHECK(w0, x1) 53fsu_fault_nopcb: 54 mov x0, #-1 55 ret 56END(fsu_fault) 57 58/* 59 * int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) 60 */ 61ENTRY(casueword32) 62 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 63 adr x6, fsu_fault /* Load the fault handler */ 64 mov w5, #1 65 SET_FAULT_HANDLER(x6, x4) /* And set it */ 66 ENTER_USER_ACCESS(w6, x4) 67 ldxr w4, [x0] /* Load-exclusive the data */ 68 cmp w4, w1 /* Compare */ 69 b.ne 1f /* Not equal, exit */ 70 stxr w5, w3, [x0] /* Store the new data */ 711: EXIT_USER_ACCESS(w6) 72 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 73 str w4, [x2] /* Store the read data */ 74 mov w0, w5 /* Result same as store status */ 75 ret /* Return */ 76END(casueword32) 77 78/* 79 * int casueword(volatile u_long *, u_long, u_long *, u_long) 80 */ 81ENTRY(casueword) 82 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 83 adr x6, fsu_fault /* Load the fault handler */ 84 mov w5, #1 85 SET_FAULT_HANDLER(x6, x4) /* And set it */ 86 ENTER_USER_ACCESS(w6, x4) 87 ldxr x4, [x0] /* Load-exclusive the data */ 88 cmp x4, x1 /* Compare */ 89 b.ne 1f /* Not equal, exit */ 90 stxr w5, x3, [x0] /* Store the new data */ 911: EXIT_USER_ACCESS(w6) 92 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 93 str x4, [x2] /* Store the read data */ 94 mov w0, w5 /* Result same as store status */ 95 ret /* Return */ 96END(casueword) 97 98.macro fsudata insn, ret_reg, user_arg 99 adr x7, fsu_fault /* Load the fault handler */ 100 SET_FAULT_HANDLER(x7, x6) /* And set it */ 101 \insn \ret_reg, [x\user_arg] /* Try accessing the data */ 102 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 103.endm 104 105/* 106 * int fubyte(volatile const void *) 107 */ 108ENTRY(fubyte) 109 check_user_access 0, (VM_MAXUSER_ADDRESS), fsu_fault_nopcb 110 fsudata ldtrb, w0, 0 111 ret /* Return */ 112END(fubyte) 113 114/* 115 * int fuword(volatile const void *) 116 */ 117ENTRY(fuword16) 118 check_user_access 0, (VM_MAXUSER_ADDRESS-1), fsu_fault_nopcb 119 fsudata ldtrh, w0, 0 120 ret /* Return */ 121END(fuword16) 122 123/* 124 * int32_t fueword32(volatile const void *, int32_t *) 125 */ 126ENTRY(fueword32) 127 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 128 fsudata ldtr, w0, 0 129 str w0, [x1] /* Save the data in kernel space */ 130 mov w0, #0 /* Success */ 131 ret /* Return */ 132END(fueword32) 133 134/* 135 * long fueword(volatile const void *, int64_t *) 136 * int64_t fueword64(volatile const void *, int64_t *) 137 */ 138ENTRY(fueword) 139EENTRY(fueword64) 140 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 141 fsudata ldtr, x0, 0 142 str x0, [x1] /* Save the data in kernel space */ 143 mov x0, #0 /* Success */ 144 ret /* Return */ 145EEND(fueword64) 146END(fueword) 147 148/* 149 * int subyte(volatile void *, int) 150 */ 151ENTRY(subyte) 152 check_user_access 0, (VM_MAXUSER_ADDRESS), fsu_fault_nopcb 153 fsudata sttrb, w1, 0 154 mov x0, #0 /* Success */ 155 ret /* Return */ 156END(subyte) 157 158/* 159 * int suword16(volatile void *, int) 160 */ 161ENTRY(suword16) 162 check_user_access 0, (VM_MAXUSER_ADDRESS-1), fsu_fault_nopcb 163 fsudata sttrh, w1, 0 164 mov x0, #0 /* Success */ 165 ret /* Return */ 166END(suword16) 167 168/* 169 * int suword32(volatile void *, int) 170 */ 171ENTRY(suword32) 172 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 173 fsudata sttr, w1, 0 174 mov x0, #0 /* Success */ 175 ret /* Return */ 176END(suword32) 177 178/* 179 * int suword(volatile void *, long) 180 */ 181ENTRY(suword) 182EENTRY(suword64) 183 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 184 fsudata sttr, x1, 0 185 mov x0, #0 /* Success */ 186 ret /* Return */ 187EEND(suword64) 188END(suword) 189 190ENTRY(setjmp) 191 /* Store the stack pointer */ 192 mov x8, sp 193 str x8, [x0], #8 194 195 /* Store the general purpose registers and lr */ 196 stp x19, x20, [x0], #16 197 stp x21, x22, [x0], #16 198 stp x23, x24, [x0], #16 199 stp x25, x26, [x0], #16 200 stp x27, x28, [x0], #16 201 stp x29, lr, [x0], #16 202 203 /* Return value */ 204 mov x0, #0 205 ret 206END(setjmp) 207 208ENTRY(longjmp) 209 /* Restore the stack pointer */ 210 ldr x8, [x0], #8 211 mov sp, x8 212 213 /* Restore the general purpose registers and lr */ 214 ldp x19, x20, [x0], #16 215 ldp x21, x22, [x0], #16 216 ldp x23, x24, [x0], #16 217 ldp x25, x26, [x0], #16 218 ldp x27, x28, [x0], #16 219 ldp x29, lr, [x0], #16 220 221 /* Load the return value */ 222 mov x0, x1 223 ret 224END(longjmp) 225 226/* 227 * pagezero, simple implementation 228 */ 229ENTRY(pagezero_simple) 230 add x1, x0, #PAGE_SIZE 231 2321: 233 stp xzr, xzr, [x0], #0x10 234 stp xzr, xzr, [x0], #0x10 235 stp xzr, xzr, [x0], #0x10 236 stp xzr, xzr, [x0], #0x10 237 cmp x0, x1 238 b.ne 1b 239 ret 240 241END(pagezero_simple) 242 243/* 244 * pagezero, cache assisted 245 */ 246ENTRY(pagezero_cache) 247 add x1, x0, #PAGE_SIZE 248 249 ldr x2, =dczva_line_size 250 ldr x2, [x2] 251 2521: 253 dc zva, x0 254 add x0, x0, x2 255 cmp x0, x1 256 b.ne 1b 257 ret 258 259END(pagezero_cache) 260