1 /*-
2 * Copyright (c) 2015-2018 Ruslan Bukin <[email protected]>
3 * All rights reserved.
4 *
5 * Portions of this software were developed by SRI International and the
6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract
7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
8 *
9 * Portions of this software were developed by the University of Cambridge
10 * Computer Laboratory as part of the CTSRD Project, with support from the
11 * UK Higher Education Innovation Fund (HEIF).
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
37
38 #include <sys/param.h>
39 #include <sys/systm.h>
40 #include <sys/kernel.h>
41 #include <sys/ktr.h>
42 #include <sys/lock.h>
43 #include <sys/mutex.h>
44 #include <sys/bus.h>
45 #include <sys/proc.h>
46 #include <sys/ptrace.h>
47 #include <sys/syscall.h>
48 #include <sys/sysent.h>
49 #ifdef KDB
50 #include <sys/kdb.h>
51 #endif
52
53 #include <vm/vm.h>
54 #include <vm/pmap.h>
55 #include <vm/vm_kern.h>
56 #include <vm/vm_map.h>
57 #include <vm/vm_param.h>
58 #include <vm/vm_extern.h>
59
60 #ifdef FPE
61 #include <machine/fpe.h>
62 #endif
63 #include <machine/frame.h>
64 #include <machine/pcb.h>
65 #include <machine/pcpu.h>
66
67 #include <machine/resource.h>
68 #include <machine/intr.h>
69
70 #ifdef KDTRACE_HOOKS
71 #include <sys/dtrace_bsd.h>
72 #endif
73
74 int (*dtrace_invop_jump_addr)(struct trapframe *);
75
76 /* Called from exception.S */
77 void do_trap_supervisor(struct trapframe *);
78 void do_trap_user(struct trapframe *);
79
80 static __inline void
call_trapsignal(struct thread * td,int sig,int code,void * addr,int trapno)81 call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno)
82 {
83 ksiginfo_t ksi;
84
85 ksiginfo_init_trap(&ksi);
86 ksi.ksi_signo = sig;
87 ksi.ksi_code = code;
88 ksi.ksi_addr = addr;
89 ksi.ksi_trapno = trapno;
90 trapsignal(td, &ksi);
91 }
92
93 int
cpu_fetch_syscall_args(struct thread * td)94 cpu_fetch_syscall_args(struct thread *td)
95 {
96 struct proc *p;
97 register_t *ap, *dst_ap;
98 struct syscall_args *sa;
99
100 p = td->td_proc;
101 sa = &td->td_sa;
102 ap = &td->td_frame->tf_a[0];
103 dst_ap = &sa->args[0];
104
105 sa->code = td->td_frame->tf_t[0];
106
107 if (__predict_false(sa->code == SYS_syscall || sa->code == SYS___syscall)) {
108 sa->code = *ap++;
109 } else {
110 *dst_ap++ = *ap++;
111 }
112
113 if (__predict_false(sa->code >= p->p_sysent->sv_size))
114 sa->callp = &p->p_sysent->sv_table[0];
115 else
116 sa->callp = &p->p_sysent->sv_table[sa->code];
117
118 KASSERT(sa->callp->sy_narg <= nitems(sa->args),
119 ("Syscall %d takes too many arguments", sa->code));
120
121 memcpy(dst_ap, ap, (NARGREG - 1) * sizeof(register_t));
122
123 td->td_retval[0] = 0;
124 td->td_retval[1] = 0;
125
126 return (0);
127 }
128
129 #include "../../kern/subr_syscall.c"
130
131 static void
dump_regs(struct trapframe * frame)132 dump_regs(struct trapframe *frame)
133 {
134 int n;
135 int i;
136
137 n = nitems(frame->tf_t);
138 for (i = 0; i < n; i++)
139 printf("t[%d] == 0x%016lx\n", i, frame->tf_t[i]);
140
141 n = nitems(frame->tf_s);
142 for (i = 0; i < n; i++)
143 printf("s[%d] == 0x%016lx\n", i, frame->tf_s[i]);
144
145 n = nitems(frame->tf_a);
146 for (i = 0; i < n; i++)
147 printf("a[%d] == 0x%016lx\n", i, frame->tf_a[i]);
148
149 printf("ra == 0x%016lx\n", frame->tf_ra);
150 printf("sp == 0x%016lx\n", frame->tf_sp);
151 printf("gp == 0x%016lx\n", frame->tf_gp);
152 printf("tp == 0x%016lx\n", frame->tf_tp);
153
154 printf("sepc == 0x%016lx\n", frame->tf_sepc);
155 printf("sstatus == 0x%016lx\n", frame->tf_sstatus);
156 }
157
158 static void
ecall_handler(void)159 ecall_handler(void)
160 {
161 struct thread *td;
162
163 td = curthread;
164
165 syscallenter(td);
166 syscallret(td);
167 }
168
169 static void
page_fault_handler(struct trapframe * frame,int usermode)170 page_fault_handler(struct trapframe *frame, int usermode)
171 {
172 struct vm_map *map;
173 uint64_t stval;
174 struct thread *td;
175 struct pcb *pcb;
176 vm_prot_t ftype;
177 vm_offset_t va;
178 struct proc *p;
179 int error, sig, ucode;
180 #ifdef KDB
181 bool handled;
182 #endif
183
184 #ifdef KDB
185 if (kdb_active) {
186 kdb_reenter();
187 return;
188 }
189 #endif
190
191 td = curthread;
192 p = td->td_proc;
193 pcb = td->td_pcb;
194 stval = frame->tf_stval;
195
196 if (td->td_critnest != 0 || td->td_intr_nesting_level != 0 ||
197 WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
198 "Kernel page fault") != 0)
199 goto fatal;
200
201 if (usermode) {
202 if (!VIRT_IS_VALID(stval)) {
203 call_trapsignal(td, SIGSEGV, SEGV_MAPERR, (void *)stval,
204 frame->tf_scause & SCAUSE_CODE);
205 goto done;
206 }
207 map = &td->td_proc->p_vmspace->vm_map;
208 } else {
209 /*
210 * Enable interrupts for the duration of the page fault. For
211 * user faults this was done already in do_trap_user().
212 */
213 intr_enable();
214
215 if (!VIRT_IS_VALID(stval))
216 goto fatal;
217
218 if (stval >= VM_MAX_USER_ADDRESS) {
219 map = kernel_map;
220 } else {
221 if (pcb->pcb_onfault == 0)
222 goto fatal;
223 map = &td->td_proc->p_vmspace->vm_map;
224 }
225 }
226
227 va = trunc_page(stval);
228
229 if (frame->tf_scause == SCAUSE_STORE_PAGE_FAULT) {
230 ftype = VM_PROT_WRITE;
231 } else if (frame->tf_scause == SCAUSE_INST_PAGE_FAULT) {
232 ftype = VM_PROT_EXECUTE;
233 } else {
234 ftype = VM_PROT_READ;
235 }
236
237 if (pmap_fault(map->pmap, va, ftype))
238 goto done;
239
240 error = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &sig, &ucode);
241 if (error != KERN_SUCCESS) {
242 if (usermode) {
243 call_trapsignal(td, sig, ucode, (void *)stval,
244 frame->tf_scause & SCAUSE_CODE);
245 } else {
246 if (pcb->pcb_onfault != 0) {
247 frame->tf_a[0] = error;
248 frame->tf_sepc = pcb->pcb_onfault;
249 return;
250 }
251 goto fatal;
252 }
253 }
254
255 done:
256 if (usermode)
257 userret(td, frame);
258 return;
259
260 fatal:
261 dump_regs(frame);
262 #ifdef KDB
263 if (debugger_on_trap) {
264 kdb_why = KDB_WHY_TRAP;
265 handled = kdb_trap(frame->tf_scause & SCAUSE_CODE, 0, frame);
266 kdb_why = KDB_WHY_UNSET;
267 if (handled)
268 return;
269 }
270 #endif
271 panic("Fatal page fault at %#lx: %#016lx", frame->tf_sepc, stval);
272 }
273
274 void
do_trap_supervisor(struct trapframe * frame)275 do_trap_supervisor(struct trapframe *frame)
276 {
277 uint64_t exception;
278
279 /* Ensure we came from supervisor mode, interrupts disabled */
280 KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) ==
281 SSTATUS_SPP, ("Came from S mode with interrupts enabled"));
282
283 KASSERT((csr_read(sstatus) & (SSTATUS_SUM)) == 0,
284 ("Came from S mode with SUM enabled"));
285
286 exception = frame->tf_scause & SCAUSE_CODE;
287 if ((frame->tf_scause & SCAUSE_INTR) != 0) {
288 /* Interrupt */
289 riscv_cpu_intr(frame);
290 return;
291 }
292
293 #ifdef KDTRACE_HOOKS
294 if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception))
295 return;
296 #endif
297
298 CTR3(KTR_TRAP, "do_trap_supervisor: curthread: %p, sepc: %lx, frame: %p",
299 curthread, frame->tf_sepc, frame);
300
301 switch (exception) {
302 case SCAUSE_LOAD_ACCESS_FAULT:
303 case SCAUSE_STORE_ACCESS_FAULT:
304 case SCAUSE_INST_ACCESS_FAULT:
305 dump_regs(frame);
306 panic("Memory access exception at 0x%016lx\n", frame->tf_sepc);
307 break;
308 case SCAUSE_STORE_PAGE_FAULT:
309 case SCAUSE_LOAD_PAGE_FAULT:
310 case SCAUSE_INST_PAGE_FAULT:
311 page_fault_handler(frame, 0);
312 break;
313 case SCAUSE_BREAKPOINT:
314 #ifdef KDTRACE_HOOKS
315 if (dtrace_invop_jump_addr != NULL &&
316 dtrace_invop_jump_addr(frame) == 0)
317 break;
318 #endif
319 #ifdef KDB
320 kdb_trap(exception, 0, frame);
321 #else
322 dump_regs(frame);
323 panic("No debugger in kernel.\n");
324 #endif
325 break;
326 case SCAUSE_ILLEGAL_INSTRUCTION:
327 dump_regs(frame);
328 panic("Illegal instruction at 0x%016lx\n", frame->tf_sepc);
329 break;
330 default:
331 dump_regs(frame);
332 panic("Unknown kernel exception %lx trap value %lx\n",
333 exception, frame->tf_stval);
334 }
335 }
336
337 void
do_trap_user(struct trapframe * frame)338 do_trap_user(struct trapframe *frame)
339 {
340 uint64_t exception;
341 struct thread *td;
342 struct pcb *pcb;
343
344 td = curthread;
345 pcb = td->td_pcb;
346
347 KASSERT(td->td_frame == frame,
348 ("%s: td_frame %p != frame %p", __func__, td->td_frame, frame));
349
350 /* Ensure we came from usermode, interrupts disabled */
351 KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == 0,
352 ("Came from U mode with interrupts enabled"));
353
354 KASSERT((csr_read(sstatus) & (SSTATUS_SUM)) == 0,
355 ("Came from U mode with SUM enabled"));
356
357 exception = frame->tf_scause & SCAUSE_CODE;
358 if ((frame->tf_scause & SCAUSE_INTR) != 0) {
359 /* Interrupt */
360 riscv_cpu_intr(frame);
361 return;
362 }
363 intr_enable();
364
365 CTR3(KTR_TRAP, "do_trap_user: curthread: %p, sepc: %lx, frame: %p",
366 curthread, frame->tf_sepc, frame);
367
368 switch (exception) {
369 case SCAUSE_LOAD_ACCESS_FAULT:
370 case SCAUSE_STORE_ACCESS_FAULT:
371 case SCAUSE_INST_ACCESS_FAULT:
372 call_trapsignal(td, SIGBUS, BUS_ADRERR, (void *)frame->tf_sepc,
373 exception);
374 userret(td, frame);
375 break;
376 case SCAUSE_STORE_PAGE_FAULT:
377 case SCAUSE_LOAD_PAGE_FAULT:
378 case SCAUSE_INST_PAGE_FAULT:
379 page_fault_handler(frame, 1);
380 break;
381 case SCAUSE_ECALL_USER:
382 frame->tf_sepc += 4; /* Next instruction */
383 ecall_handler();
384 break;
385 case SCAUSE_ILLEGAL_INSTRUCTION:
386 #ifdef FPE
387 if ((pcb->pcb_fpflags & PCB_FP_STARTED) == 0) {
388 /*
389 * May be a FPE trap. Enable FPE usage
390 * for this thread and try again.
391 */
392 fpe_state_clear();
393 frame->tf_sstatus &= ~SSTATUS_FS_MASK;
394 frame->tf_sstatus |= SSTATUS_FS_CLEAN;
395 pcb->pcb_fpflags |= PCB_FP_STARTED;
396 break;
397 }
398 #endif
399 call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_sepc,
400 exception);
401 userret(td, frame);
402 break;
403 case SCAUSE_BREAKPOINT:
404 call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_sepc,
405 exception);
406 userret(td, frame);
407 break;
408 default:
409 dump_regs(frame);
410 panic("Unknown userland exception %lx, trap value %lx\n",
411 exception, frame->tf_stval);
412 }
413 }
414