1 /*-
2 * Copyright (c) 2014 Andrew Turner
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
34 #include <sys/ktr.h>
35 #include <sys/lock.h>
36 #include <sys/mutex.h>
37 #include <sys/proc.h>
38 #include <sys/ptrace.h>
39 #include <sys/syscall.h>
40 #include <sys/sysent.h>
41 #ifdef KDB
42 #include <sys/kdb.h>
43 #endif
44
45 #include <vm/vm.h>
46 #include <vm/pmap.h>
47 #include <vm/vm_kern.h>
48 #include <vm/vm_map.h>
49 #include <vm/vm_param.h>
50 #include <vm/vm_extern.h>
51
52 #include <machine/frame.h>
53 #include <machine/md_var.h>
54 #include <machine/pcb.h>
55 #include <machine/pcpu.h>
56 #include <machine/undefined.h>
57
58 #ifdef KDTRACE_HOOKS
59 #include <sys/dtrace_bsd.h>
60 #endif
61
62 #ifdef VFP
63 #include <machine/vfp.h>
64 #endif
65
66 #ifdef KDB
67 #include <machine/db_machdep.h>
68 #endif
69
70 #ifdef DDB
71 #include <ddb/db_output.h>
72 #endif
73
74 extern register_t fsu_intr_fault;
75
76 /* Called from exception.S */
77 void do_el1h_sync(struct thread *, struct trapframe *);
78 void do_el0_sync(struct thread *, struct trapframe *);
79 void do_el0_error(struct trapframe *);
80 void do_serror(struct trapframe *);
81 void unhandled_exception(struct trapframe *);
82
83 static void print_registers(struct trapframe *frame);
84
85 int (*dtrace_invop_jump_addr)(struct trapframe *);
86
87 typedef void (abort_handler)(struct thread *, struct trapframe *, uint64_t,
88 uint64_t, int);
89
90 static abort_handler align_abort;
91 static abort_handler data_abort;
92 static abort_handler external_abort;
93
94 static abort_handler *abort_handlers[] = {
95 [ISS_DATA_DFSC_TF_L0] = data_abort,
96 [ISS_DATA_DFSC_TF_L1] = data_abort,
97 [ISS_DATA_DFSC_TF_L2] = data_abort,
98 [ISS_DATA_DFSC_TF_L3] = data_abort,
99 [ISS_DATA_DFSC_AFF_L1] = data_abort,
100 [ISS_DATA_DFSC_AFF_L2] = data_abort,
101 [ISS_DATA_DFSC_AFF_L3] = data_abort,
102 [ISS_DATA_DFSC_PF_L1] = data_abort,
103 [ISS_DATA_DFSC_PF_L2] = data_abort,
104 [ISS_DATA_DFSC_PF_L3] = data_abort,
105 [ISS_DATA_DFSC_ALIGN] = align_abort,
106 [ISS_DATA_DFSC_EXT] = external_abort,
107 };
108
109 static __inline void
call_trapsignal(struct thread * td,int sig,int code,void * addr,int trapno)110 call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno)
111 {
112 ksiginfo_t ksi;
113
114 ksiginfo_init_trap(&ksi);
115 ksi.ksi_signo = sig;
116 ksi.ksi_code = code;
117 ksi.ksi_addr = addr;
118 ksi.ksi_trapno = trapno;
119 trapsignal(td, &ksi);
120 }
121
122 int
cpu_fetch_syscall_args(struct thread * td)123 cpu_fetch_syscall_args(struct thread *td)
124 {
125 struct proc *p;
126 register_t *ap, *dst_ap;
127 struct syscall_args *sa;
128
129 p = td->td_proc;
130 sa = &td->td_sa;
131 ap = td->td_frame->tf_x;
132 dst_ap = &sa->args[0];
133
134 sa->code = td->td_frame->tf_x[8];
135
136 if (__predict_false(sa->code == SYS_syscall || sa->code == SYS___syscall)) {
137 sa->code = *ap++;
138 } else {
139 *dst_ap++ = *ap++;
140 }
141
142 if (__predict_false(sa->code >= p->p_sysent->sv_size))
143 sa->callp = &p->p_sysent->sv_table[0];
144 else
145 sa->callp = &p->p_sysent->sv_table[sa->code];
146
147 KASSERT(sa->callp->sy_narg <= nitems(sa->args),
148 ("Syscall %d takes too many arguments", sa->code));
149
150 memcpy(dst_ap, ap, (MAXARGS - 1) * sizeof(register_t));
151
152 td->td_retval[0] = 0;
153 td->td_retval[1] = 0;
154
155 return (0);
156 }
157
158 #include "../../kern/subr_syscall.c"
159
160 /*
161 * Test for fault generated by given access instruction in
162 * bus_peek_<foo> or bus_poke_<foo> bus function.
163 */
164 extern uint32_t generic_bs_peek_1f, generic_bs_peek_2f;
165 extern uint32_t generic_bs_peek_4f, generic_bs_peek_8f;
166 extern uint32_t generic_bs_poke_1f, generic_bs_poke_2f;
167 extern uint32_t generic_bs_poke_4f, generic_bs_poke_8f;
168
169 static bool
test_bs_fault(void * addr)170 test_bs_fault(void *addr)
171 {
172 return (addr == &generic_bs_peek_1f ||
173 addr == &generic_bs_peek_2f ||
174 addr == &generic_bs_peek_4f ||
175 addr == &generic_bs_peek_8f ||
176 addr == &generic_bs_poke_1f ||
177 addr == &generic_bs_poke_2f ||
178 addr == &generic_bs_poke_4f ||
179 addr == &generic_bs_poke_8f);
180 }
181
182 static void
svc_handler(struct thread * td,struct trapframe * frame)183 svc_handler(struct thread *td, struct trapframe *frame)
184 {
185
186 if ((frame->tf_esr & ESR_ELx_ISS_MASK) == 0) {
187 syscallenter(td);
188 syscallret(td);
189 } else {
190 call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr,
191 ESR_ELx_EXCEPTION(frame->tf_esr));
192 userret(td, frame);
193 }
194 }
195
196 static void
align_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)197 align_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
198 uint64_t far, int lower)
199 {
200 if (!lower) {
201 print_registers(frame);
202 printf(" far: %16lx\n", far);
203 printf(" esr: %.8lx\n", esr);
204 panic("Misaligned access from kernel space!");
205 }
206
207 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr,
208 ESR_ELx_EXCEPTION(frame->tf_esr));
209 userret(td, frame);
210 }
211
212
213 static void
external_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)214 external_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
215 uint64_t far, int lower)
216 {
217
218 /*
219 * Try to handle synchronous external aborts caused by
220 * bus_space_peek() and/or bus_space_poke() functions.
221 */
222 if (!lower && test_bs_fault((void *)frame->tf_elr)) {
223 frame->tf_elr = (uint64_t)generic_bs_fault;
224 return;
225 }
226
227 print_registers(frame);
228 printf(" far: %16lx\n", far);
229 panic("Unhandled EL%d external data abort", lower ? 0: 1);
230 }
231
232 static void
data_abort(struct thread * td,struct trapframe * frame,uint64_t esr,uint64_t far,int lower)233 data_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
234 uint64_t far, int lower)
235 {
236 struct vm_map *map;
237 struct proc *p;
238 struct pcb *pcb;
239 vm_prot_t ftype;
240 int error, sig, ucode;
241 #ifdef KDB
242 bool handled;
243 #endif
244
245 /*
246 * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive
247 * and Store-Exclusive instruction usage restrictions", state
248 * of the exclusive monitors after data abort exception is unknown.
249 */
250 clrex();
251
252 #ifdef KDB
253 if (kdb_active) {
254 kdb_reenter();
255 return;
256 }
257 #endif
258
259 pcb = td->td_pcb;
260 p = td->td_proc;
261 if (lower)
262 map = &p->p_vmspace->vm_map;
263 else {
264 intr_enable();
265
266 /* The top bit tells us which range to use */
267 if (far >= VM_MAXUSER_ADDRESS) {
268 map = kernel_map;
269 } else {
270 map = &p->p_vmspace->vm_map;
271 if (map == NULL)
272 map = kernel_map;
273 }
274 }
275
276 /*
277 * Try to handle translation, access flag, and permission faults.
278 * Translation faults may occur as a result of the required
279 * break-before-make sequence used when promoting or demoting
280 * superpages. Such faults must not occur while holding the pmap lock,
281 * or pmap_fault() will recurse on that lock.
282 */
283 if ((lower || map == kernel_map || pcb->pcb_onfault != 0) &&
284 pmap_fault(map->pmap, esr, far) == KERN_SUCCESS)
285 return;
286
287 KASSERT(td->td_md.md_spinlock_count == 0,
288 ("data abort with spinlock held"));
289 if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK |
290 WARN_GIANTOK, NULL, "Kernel page fault") != 0) {
291 print_registers(frame);
292 printf(" far: %16lx\n", far);
293 printf(" esr: %.8lx\n", esr);
294 panic("data abort in critical section or under mutex");
295 }
296
297 switch (ESR_ELx_EXCEPTION(esr)) {
298 case EXCP_INSN_ABORT:
299 case EXCP_INSN_ABORT_L:
300 ftype = VM_PROT_EXECUTE;
301 break;
302 default:
303 ftype = (esr & ISS_DATA_WnR) == 0 ? VM_PROT_READ :
304 VM_PROT_WRITE;
305 break;
306 }
307
308 /* Fault in the page. */
309 error = vm_fault_trap(map, far, ftype, VM_FAULT_NORMAL, &sig, &ucode);
310 if (error != KERN_SUCCESS) {
311 if (lower) {
312 call_trapsignal(td, sig, ucode, (void *)far,
313 ESR_ELx_EXCEPTION(esr));
314 } else {
315 if (td->td_intr_nesting_level == 0 &&
316 pcb->pcb_onfault != 0) {
317 frame->tf_x[0] = error;
318 frame->tf_elr = pcb->pcb_onfault;
319 return;
320 }
321
322 printf("Fatal data abort:\n");
323 print_registers(frame);
324 printf(" far: %16lx\n", far);
325 printf(" esr: %.8lx\n", esr);
326
327 #ifdef KDB
328 if (debugger_on_trap) {
329 kdb_why = KDB_WHY_TRAP;
330 handled = kdb_trap(ESR_ELx_EXCEPTION(esr), 0,
331 frame);
332 kdb_why = KDB_WHY_UNSET;
333 if (handled)
334 return;
335 }
336 #endif
337 panic("vm_fault failed: %lx", frame->tf_elr);
338 }
339 }
340
341 if (lower)
342 userret(td, frame);
343 }
344
345 static void
print_registers(struct trapframe * frame)346 print_registers(struct trapframe *frame)
347 {
348 u_int reg;
349
350 for (reg = 0; reg < nitems(frame->tf_x); reg++) {
351 printf(" %sx%d: %16lx\n", (reg < 10) ? " " : "", reg,
352 frame->tf_x[reg]);
353 }
354 printf(" sp: %16lx\n", frame->tf_sp);
355 printf(" lr: %16lx\n", frame->tf_lr);
356 printf(" elr: %16lx\n", frame->tf_elr);
357 printf("spsr: %8x\n", frame->tf_spsr);
358 }
359
360 void
do_el1h_sync(struct thread * td,struct trapframe * frame)361 do_el1h_sync(struct thread *td, struct trapframe *frame)
362 {
363 uint32_t exception;
364 uint64_t esr, far;
365 int dfsc;
366
367 /* Read the esr register to get the exception details */
368 esr = frame->tf_esr;
369 exception = ESR_ELx_EXCEPTION(esr);
370
371 #ifdef KDTRACE_HOOKS
372 if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception))
373 return;
374 #endif
375
376 CTR4(KTR_TRAP,
377 "do_el1_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td,
378 esr, frame->tf_elr, frame);
379
380 switch (exception) {
381 case EXCP_FP_SIMD:
382 case EXCP_TRAP_FP:
383 #ifdef VFP
384 if ((td->td_pcb->pcb_fpflags & PCB_FP_KERN) != 0) {
385 vfp_restore_state();
386 } else
387 #endif
388 {
389 print_registers(frame);
390 printf(" esr: %.8lx\n", esr);
391 panic("VFP exception in the kernel");
392 }
393 break;
394 case EXCP_INSN_ABORT:
395 case EXCP_DATA_ABORT:
396 far = READ_SPECIALREG(far_el1);
397 dfsc = esr & ISS_DATA_DFSC_MASK;
398 if (dfsc < nitems(abort_handlers) &&
399 abort_handlers[dfsc] != NULL) {
400 abort_handlers[dfsc](td, frame, esr, far, 0);
401 } else {
402 print_registers(frame);
403 printf(" far: %16lx\n", far);
404 printf(" esr: %.8lx\n", esr);
405 panic("Unhandled EL1 %s abort: %x",
406 exception == EXCP_INSN_ABORT ? "instruction" :
407 "data", dfsc);
408 }
409 break;
410 case EXCP_BRK:
411 #ifdef KDTRACE_HOOKS
412 if ((esr & ESR_ELx_ISS_MASK) == 0x40d && \
413 dtrace_invop_jump_addr != 0) {
414 dtrace_invop_jump_addr(frame);
415 break;
416 }
417 #endif
418 #ifdef KDB
419 kdb_trap(exception, 0, frame);
420 #else
421 panic("No debugger in kernel.\n");
422 #endif
423 break;
424 case EXCP_WATCHPT_EL1:
425 case EXCP_SOFTSTP_EL1:
426 #ifdef KDB
427 kdb_trap(exception, 0, frame);
428 #else
429 panic("No debugger in kernel.\n");
430 #endif
431 break;
432 case EXCP_UNKNOWN:
433 if (undef_insn(1, frame))
434 break;
435 /* FALLTHROUGH */
436 default:
437 print_registers(frame);
438 printf(" far: %16lx\n", READ_SPECIALREG(far_el1));
439 panic("Unknown kernel exception %x esr_el1 %lx\n", exception,
440 esr);
441 }
442 }
443
444 void
do_el0_sync(struct thread * td,struct trapframe * frame)445 do_el0_sync(struct thread *td, struct trapframe *frame)
446 {
447 pcpu_bp_harden bp_harden;
448 uint32_t exception;
449 uint64_t esr, far;
450 int dfsc;
451
452 /* Check we have a sane environment when entering from userland */
453 KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS,
454 ("Invalid pcpu address from userland: %p (tpidr %lx)",
455 get_pcpu(), READ_SPECIALREG(tpidr_el1)));
456
457 esr = frame->tf_esr;
458 exception = ESR_ELx_EXCEPTION(esr);
459 switch (exception) {
460 case EXCP_INSN_ABORT_L:
461 far = READ_SPECIALREG(far_el1);
462
463 /*
464 * Userspace may be trying to train the branch predictor to
465 * attack the kernel. If we are on a CPU affected by this
466 * call the handler to clear the branch predictor state.
467 */
468 if (far > VM_MAXUSER_ADDRESS) {
469 bp_harden = PCPU_GET(bp_harden);
470 if (bp_harden != NULL)
471 bp_harden();
472 }
473 break;
474 case EXCP_UNKNOWN:
475 case EXCP_DATA_ABORT_L:
476 case EXCP_DATA_ABORT:
477 case EXCP_WATCHPT_EL0:
478 far = READ_SPECIALREG(far_el1);
479 break;
480 }
481 intr_enable();
482
483 CTR4(KTR_TRAP,
484 "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, esr,
485 frame->tf_elr, frame);
486
487 switch (exception) {
488 case EXCP_FP_SIMD:
489 case EXCP_TRAP_FP:
490 #ifdef VFP
491 vfp_restore_state();
492 #else
493 panic("VFP exception in userland");
494 #endif
495 break;
496 case EXCP_SVC32:
497 case EXCP_SVC64:
498 svc_handler(td, frame);
499 break;
500 case EXCP_INSN_ABORT_L:
501 case EXCP_DATA_ABORT_L:
502 case EXCP_DATA_ABORT:
503 dfsc = esr & ISS_DATA_DFSC_MASK;
504 if (dfsc < nitems(abort_handlers) &&
505 abort_handlers[dfsc] != NULL)
506 abort_handlers[dfsc](td, frame, esr, far, 1);
507 else {
508 print_registers(frame);
509 printf(" far: %16lx\n", far);
510 printf(" esr: %.8lx\n", esr);
511 panic("Unhandled EL0 %s abort: %x",
512 exception == EXCP_INSN_ABORT_L ? "instruction" :
513 "data", dfsc);
514 }
515 break;
516 case EXCP_UNKNOWN:
517 if (!undef_insn(0, frame))
518 call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)far,
519 exception);
520 userret(td, frame);
521 break;
522 case EXCP_SP_ALIGN:
523 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sp,
524 exception);
525 userret(td, frame);
526 break;
527 case EXCP_PC_ALIGN:
528 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr,
529 exception);
530 userret(td, frame);
531 break;
532 case EXCP_BRKPT_EL0:
533 case EXCP_BRK:
534 call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_elr,
535 exception);
536 userret(td, frame);
537 break;
538 case EXCP_WATCHPT_EL0:
539 call_trapsignal(td, SIGTRAP, TRAP_TRACE, (void *)far,
540 exception);
541 userret(td, frame);
542 break;
543 case EXCP_MSR:
544 /*
545 * The CPU can raise EXCP_MSR when userspace executes an mrs
546 * instruction to access a special register userspace doesn't
547 * have access to.
548 */
549 if (!undef_insn(0, frame))
550 call_trapsignal(td, SIGILL, ILL_PRVOPC,
551 (void *)frame->tf_elr, exception);
552 userret(td, frame);
553 break;
554 case EXCP_SOFTSTP_EL0:
555 td->td_frame->tf_spsr &= ~PSR_SS;
556 td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP;
557 WRITE_SPECIALREG(mdscr_el1,
558 READ_SPECIALREG(mdscr_el1) & ~DBG_MDSCR_SS);
559 call_trapsignal(td, SIGTRAP, TRAP_TRACE,
560 (void *)frame->tf_elr, exception);
561 userret(td, frame);
562 break;
563 default:
564 call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr,
565 exception);
566 userret(td, frame);
567 break;
568 }
569
570 KASSERT((td->td_pcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0,
571 ("Kernel VFP flags set while entering userspace"));
572 KASSERT(
573 td->td_pcb->pcb_fpusaved == &td->td_pcb->pcb_fpustate,
574 ("Kernel VFP state in use when entering userspace"));
575 }
576
577 /*
578 * TODO: We will need to handle these later when we support ARMv8.2 RAS.
579 */
580 void
do_serror(struct trapframe * frame)581 do_serror(struct trapframe *frame)
582 {
583 uint64_t esr, far;
584
585 far = READ_SPECIALREG(far_el1);
586 esr = frame->tf_esr;
587
588 print_registers(frame);
589 printf(" far: %16lx\n", far);
590 printf(" esr: %.8lx\n", esr);
591 panic("Unhandled System Error");
592 }
593
594 void
unhandled_exception(struct trapframe * frame)595 unhandled_exception(struct trapframe *frame)
596 {
597 uint64_t esr, far;
598
599 far = READ_SPECIALREG(far_el1);
600 esr = frame->tf_esr;
601
602 print_registers(frame);
603 printf(" far: %16lx\n", far);
604 printf(" esr: %.8lx\n", esr);
605 panic("Unhandled exception");
606 }
607