1 /* $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $ */
2
3 /*
4 * Copyright (c) 2019 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 * Copyright (c) 2019 Andrew Turner
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Maxime Villard.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #define SAN_RUNTIME
34
35 #include "opt_ddb.h"
36
37 #include <sys/cdefs.h>
38 #include <sys/param.h>
39 #include <sys/kernel.h>
40 #include <sys/bus.h>
41 #include <sys/conf.h>
42 #include <sys/cpu.h>
43 #include <sys/csan.h>
44 #include <sys/proc.h>
45 #include <sys/smp.h>
46 #include <sys/systm.h>
47
48 #include <ddb/ddb.h>
49 #include <ddb/db_sym.h>
50
51 #ifdef KCSAN_PANIC
52 #define REPORT panic
53 #else
54 #define REPORT printf
55 #endif
56
57 typedef struct {
58 uintptr_t addr;
59 uint32_t size;
60 bool write:1;
61 bool atomic:1;
62 uintptr_t pc;
63 } csan_cell_t;
64
65 typedef struct {
66 bool inited;
67 uint32_t cnt;
68 csan_cell_t cell;
69 } csan_cpu_t;
70
71 static csan_cpu_t kcsan_cpus[MAXCPU];
72 static bool kcsan_enabled __read_mostly;
73
74 #define __RET_ADDR (uintptr_t)__builtin_return_address(0)
75
76 #define KCSAN_NACCESSES 1024
77 #define KCSAN_DELAY 10 /* 10 microseconds */
78
79 /* -------------------------------------------------------------------------- */
80
81 /* The MD code. */
82 #include <machine/csan.h>
83
84 /* -------------------------------------------------------------------------- */
85
86 static void
kcsan_enable(void * dummy __unused)87 kcsan_enable(void *dummy __unused)
88 {
89
90 printf("Enabling KCSCAN, expect reduced performance.\n");
91 kcsan_enabled = true;
92 }
93 SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL);
94
95 void
kcsan_cpu_init(u_int cpu)96 kcsan_cpu_init(u_int cpu)
97 {
98 kcsan_cpus[cpu].inited = true;
99 }
100
101 /* -------------------------------------------------------------------------- */
102
103 static inline void
kcsan_report(csan_cell_t * new,u_int newcpu,csan_cell_t * old,u_int oldcpu)104 kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu)
105 {
106 const char *newsym, *oldsym;
107 #ifdef DDB
108 c_db_sym_t sym;
109 db_expr_t offset;
110
111 sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset);
112 db_symbol_values(sym, &newsym, NULL);
113
114 sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset);
115 db_symbol_values(sym, &oldsym, NULL);
116 #else
117 newsym = "";
118 oldsym = "";
119 #endif
120 REPORT("CSan: Racy Access "
121 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] "
122 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n",
123 newcpu,
124 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
125 (void *)new->addr, new->size, (void *)new->pc, newsym,
126 oldcpu,
127 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"),
128 (void *)old->addr, old->size, (void *)old->pc, oldsym);
129 kcsan_md_unwind();
130 }
131
132 static inline bool
kcsan_access_is_atomic(csan_cell_t * new,csan_cell_t * old)133 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old)
134 {
135 if (new->write && !new->atomic)
136 return false;
137 if (old->write && !old->atomic)
138 return false;
139 return true;
140 }
141
142 static inline void
kcsan_access(uintptr_t addr,size_t size,bool write,bool atomic,uintptr_t pc)143 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc)
144 {
145 csan_cell_t old, new;
146 csan_cpu_t *cpu;
147 uint64_t intr;
148 size_t i;
149
150 if (__predict_false(!kcsan_enabled))
151 return;
152 if (__predict_false(kcsan_md_unsupported((vm_offset_t)addr)))
153 return;
154 if (KERNEL_PANICKED())
155 return;
156
157 new.addr = addr;
158 new.size = size;
159 new.write = write;
160 new.atomic = atomic;
161 new.pc = pc;
162
163 CPU_FOREACH(i) {
164 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
165
166 if (old.addr + old.size <= new.addr)
167 continue;
168 if (new.addr + new.size <= old.addr)
169 continue;
170 if (__predict_true(!old.write && !new.write))
171 continue;
172 if (__predict_true(kcsan_access_is_atomic(&new, &old)))
173 continue;
174
175 kcsan_report(&new, PCPU_GET(cpuid), &old, i);
176 break;
177 }
178
179 if (__predict_false(!kcsan_md_is_avail()))
180 return;
181
182 kcsan_md_disable_intrs(&intr);
183
184 cpu = &kcsan_cpus[PCPU_GET(cpuid)];
185 if (__predict_false(!cpu->inited))
186 goto out;
187 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
188 if (__predict_true(cpu->cnt != 0))
189 goto out;
190
191 __builtin_memcpy(&cpu->cell, &new, sizeof(new));
192 kcsan_md_delay(KCSAN_DELAY);
193 __builtin_memset(&cpu->cell, 0, sizeof(new));
194
195 out:
196 kcsan_md_enable_intrs(&intr);
197 }
198
199 #define CSAN_READ(size) \
200 void __tsan_read##size(uintptr_t); \
201 void __tsan_read##size(uintptr_t addr) \
202 { \
203 kcsan_access(addr, size, false, false, __RET_ADDR); \
204 } \
205 void __tsan_unaligned_read##size(uintptr_t); \
206 void __tsan_unaligned_read##size(uintptr_t addr) \
207 { \
208 kcsan_access(addr, size, false, false, __RET_ADDR); \
209 }
210
211 CSAN_READ(1)
212 CSAN_READ(2)
213 CSAN_READ(4)
214 CSAN_READ(8)
215 CSAN_READ(16)
216
217 #define CSAN_WRITE(size) \
218 void __tsan_write##size(uintptr_t); \
219 void __tsan_write##size(uintptr_t addr) \
220 { \
221 kcsan_access(addr, size, true, false, __RET_ADDR); \
222 } \
223 void __tsan_unaligned_write##size(uintptr_t); \
224 void __tsan_unaligned_write##size(uintptr_t addr) \
225 { \
226 kcsan_access(addr, size, true, false, __RET_ADDR); \
227 }
228
229 CSAN_WRITE(1)
230 CSAN_WRITE(2)
231 CSAN_WRITE(4)
232 CSAN_WRITE(8)
233 CSAN_WRITE(16)
234
235 void __tsan_read_range(uintptr_t, size_t);
236 void __tsan_write_range(uintptr_t, size_t);
237
238 void
__tsan_read_range(uintptr_t addr,size_t size)239 __tsan_read_range(uintptr_t addr, size_t size)
240 {
241 kcsan_access(addr, size, false, false, __RET_ADDR);
242 }
243
244 void
__tsan_write_range(uintptr_t addr,size_t size)245 __tsan_write_range(uintptr_t addr, size_t size)
246 {
247 kcsan_access(addr, size, true, false, __RET_ADDR);
248 }
249
250 void __tsan_init(void);
251 void __tsan_func_entry(void *);
252 void __tsan_func_exit(void);
253
254 void
__tsan_init(void)255 __tsan_init(void)
256 {
257 }
258
259 void
__tsan_func_entry(void * call_pc)260 __tsan_func_entry(void *call_pc)
261 {
262 }
263
264 void
__tsan_func_exit(void)265 __tsan_func_exit(void)
266 {
267 }
268
269 /* -------------------------------------------------------------------------- */
270
271 void *
kcsan_memcpy(void * dst,const void * src,size_t len)272 kcsan_memcpy(void *dst, const void *src, size_t len)
273 {
274 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
275 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
276 return __builtin_memcpy(dst, src, len);
277 }
278
279 int
kcsan_memcmp(const void * b1,const void * b2,size_t len)280 kcsan_memcmp(const void *b1, const void *b2, size_t len)
281 {
282 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR);
283 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR);
284 return __builtin_memcmp(b1, b2, len);
285 }
286
287 void *
kcsan_memset(void * b,int c,size_t len)288 kcsan_memset(void *b, int c, size_t len)
289 {
290 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR);
291 return __builtin_memset(b, c, len);
292 }
293
294 void *
kcsan_memmove(void * dst,const void * src,size_t len)295 kcsan_memmove(void *dst, const void *src, size_t len)
296 {
297 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
298 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
299 return __builtin_memmove(dst, src, len);
300 }
301
302 __strong_reference(kcsan_memcpy, __tsan_memcpy);
303 __strong_reference(kcsan_memset, __tsan_memset);
304 __strong_reference(kcsan_memmove, __tsan_memmove);
305
306 char *
kcsan_strcpy(char * dst,const char * src)307 kcsan_strcpy(char *dst, const char *src)
308 {
309 char *save = dst;
310
311 while (1) {
312 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR);
313 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR);
314 *dst = *src;
315 if (*src == '\0')
316 break;
317 src++, dst++;
318 }
319
320 return save;
321 }
322
323 int
kcsan_strcmp(const char * s1,const char * s2)324 kcsan_strcmp(const char *s1, const char *s2)
325 {
326 while (1) {
327 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR);
328 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR);
329 if (*s1 != *s2)
330 break;
331 if (*s1 == '\0')
332 return 0;
333 s1++, s2++;
334 }
335
336 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
337 }
338
339 size_t
kcsan_strlen(const char * str)340 kcsan_strlen(const char *str)
341 {
342 const char *s;
343
344 s = str;
345 while (1) {
346 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR);
347 if (*s == '\0')
348 break;
349 s++;
350 }
351
352 return (s - str);
353 }
354
355 int
kcsan_copyin(const void * uaddr,void * kaddr,size_t len)356 kcsan_copyin(const void *uaddr, void *kaddr, size_t len)
357 {
358 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
359 return copyin(uaddr, kaddr, len);
360 }
361
362 int
kcsan_copyinstr(const void * uaddr,void * kaddr,size_t len,size_t * done)363 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
364 {
365 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
366 return copyinstr(uaddr, kaddr, len, done);
367 }
368
369 int
kcsan_copyout(const void * kaddr,void * uaddr,size_t len)370 kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
371 {
372 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
373 return copyout(kaddr, uaddr, len);
374 }
375
376 /* -------------------------------------------------------------------------- */
377
378 #include <machine/atomic.h>
379 #include <sys/atomic_san.h>
380
381 #define _CSAN_ATOMIC_FUNC_ADD(name, type) \
382 void kcsan_atomic_add_##name(volatile type *ptr, type val) \
383 { \
384 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
385 __RET_ADDR); \
386 atomic_add_##name(ptr, val); \
387 }
388
389 #define CSAN_ATOMIC_FUNC_ADD(name, type) \
390 _CSAN_ATOMIC_FUNC_ADD(name, type) \
391 _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \
392 _CSAN_ATOMIC_FUNC_ADD(rel_##name, type)
393
394 #define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
395 void kcsan_atomic_clear_##name(volatile type *ptr, type val) \
396 { \
397 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
398 __RET_ADDR); \
399 atomic_clear_##name(ptr, val); \
400 }
401
402 #define CSAN_ATOMIC_FUNC_CLEAR(name, type) \
403 _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
404 _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
405 _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
406
407 #define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
408 int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \
409 type val2) \
410 { \
411 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
412 __RET_ADDR); \
413 return (atomic_cmpset_##name(ptr, val1, val2)); \
414 }
415
416 #define CSAN_ATOMIC_FUNC_CMPSET(name, type) \
417 _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
418 _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
419 _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
420
421 #define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
422 int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \
423 type val2) \
424 { \
425 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
426 __RET_ADDR); \
427 return (atomic_fcmpset_##name(ptr, val1, val2)); \
428 }
429
430 #define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
431 _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
432 _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
433 _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
434
435 #define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \
436 type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
437 { \
438 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
439 __RET_ADDR); \
440 return (atomic_fetchadd_##name(ptr, val)); \
441 }
442
443 #define _CSAN_ATOMIC_FUNC_LOAD(name, type) \
444 type kcsan_atomic_load_##name(volatile type *ptr) \
445 { \
446 kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \
447 __RET_ADDR); \
448 return (atomic_load_##name(ptr)); \
449 }
450
451 #define CSAN_ATOMIC_FUNC_LOAD(name, type) \
452 _CSAN_ATOMIC_FUNC_LOAD(name, type) \
453 _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \
454
455 #define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
456 type kcsan_atomic_readandclear_##name(volatile type *ptr) \
457 { \
458 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
459 __RET_ADDR); \
460 return (atomic_readandclear_##name(ptr)); \
461 }
462
463 #define _CSAN_ATOMIC_FUNC_SET(name, type) \
464 void kcsan_atomic_set_##name(volatile type *ptr, type val) \
465 { \
466 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
467 __RET_ADDR); \
468 atomic_set_##name(ptr, val); \
469 }
470
471 #define CSAN_ATOMIC_FUNC_SET(name, type) \
472 _CSAN_ATOMIC_FUNC_SET(name, type) \
473 _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \
474 _CSAN_ATOMIC_FUNC_SET(rel_##name, type)
475
476 #define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
477 void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \
478 { \
479 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
480 __RET_ADDR); \
481 atomic_subtract_##name(ptr, val); \
482 }
483
484 #define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
485 _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
486 _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
487 _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
488
489 #define _CSAN_ATOMIC_FUNC_STORE(name, type) \
490 void kcsan_atomic_store_##name(volatile type *ptr, type val) \
491 { \
492 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
493 __RET_ADDR); \
494 atomic_store_##name(ptr, val); \
495 }
496
497 #define CSAN_ATOMIC_FUNC_STORE(name, type) \
498 _CSAN_ATOMIC_FUNC_STORE(name, type) \
499 _CSAN_ATOMIC_FUNC_STORE(rel_##name, type)
500
501 #define CSAN_ATOMIC_FUNC_SWAP(name, type) \
502 type kcsan_atomic_swap_##name(volatile type *ptr, type val) \
503 { \
504 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
505 __RET_ADDR); \
506 return(atomic_swap_##name(ptr, val)); \
507 }
508
509 #define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
510 int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \
511 { \
512 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
513 __RET_ADDR); \
514 return(atomic_testandclear_##name(ptr, val)); \
515 }
516
517 #define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \
518 int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \
519 { \
520 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
521 __RET_ADDR); \
522 return (atomic_testandset_##name(ptr, val)); \
523 }
524
_CSAN_ATOMIC_FUNC_LOAD(bool,bool)525 _CSAN_ATOMIC_FUNC_LOAD(bool, bool)
526 _CSAN_ATOMIC_FUNC_STORE(bool, bool)
527
528 CSAN_ATOMIC_FUNC_ADD(8, uint8_t)
529 CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t)
530 CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t)
531 CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t)
532 CSAN_ATOMIC_FUNC_LOAD(8, uint8_t)
533 CSAN_ATOMIC_FUNC_SET(8, uint8_t)
534 CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t)
535 _CSAN_ATOMIC_FUNC_STORE(8, uint8_t)
536 #if 0
537 CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t)
538 CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t)
539 CSAN_ATOMIC_FUNC_SWAP(8, uint8_t)
540 CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t)
541 CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t)
542 #endif
543
544 CSAN_ATOMIC_FUNC_ADD(16, uint16_t)
545 CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t)
546 CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t)
547 CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t)
548 CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
549 CSAN_ATOMIC_FUNC_SET(16, uint16_t)
550 CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t)
551 _CSAN_ATOMIC_FUNC_STORE(16, uint16_t)
552 #if 0
553 CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t)
554 CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t)
555 CSAN_ATOMIC_FUNC_SWAP(16, uint16_t)
556 CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t)
557 CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t)
558 #endif
559
560 CSAN_ATOMIC_FUNC_ADD(32, uint32_t)
561 CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t)
562 CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t)
563 CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t)
564 CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t)
565 CSAN_ATOMIC_FUNC_LOAD(32, uint32_t)
566 CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t)
567 CSAN_ATOMIC_FUNC_SET(32, uint32_t)
568 CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t)
569 CSAN_ATOMIC_FUNC_STORE(32, uint32_t)
570 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t)
571 CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t)
572 CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t)
573
574 CSAN_ATOMIC_FUNC_ADD(64, uint64_t)
575 CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t)
576 CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t)
577 CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t)
578 CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t)
579 CSAN_ATOMIC_FUNC_LOAD(64, uint64_t)
580 CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t)
581 CSAN_ATOMIC_FUNC_SET(64, uint64_t)
582 CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t)
583 CSAN_ATOMIC_FUNC_STORE(64, uint64_t)
584 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t)
585 CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t)
586 CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t)
587
588 CSAN_ATOMIC_FUNC_ADD(char, uint8_t)
589 CSAN_ATOMIC_FUNC_CLEAR(char, uint8_t)
590 CSAN_ATOMIC_FUNC_CMPSET(char, uint8_t)
591 CSAN_ATOMIC_FUNC_FCMPSET(char, uint8_t)
592 CSAN_ATOMIC_FUNC_LOAD(char, uint8_t)
593 CSAN_ATOMIC_FUNC_SET(char, uint8_t)
594 CSAN_ATOMIC_FUNC_SUBTRACT(char, uint8_t)
595 _CSAN_ATOMIC_FUNC_STORE(char, uint8_t)
596 #if 0
597 CSAN_ATOMIC_FUNC_FETCHADD(char, uint8_t)
598 CSAN_ATOMIC_FUNC_READANDCLEAR(char, uint8_t)
599 CSAN_ATOMIC_FUNC_SWAP(char, uint8_t)
600 CSAN_ATOMIC_FUNC_TESTANDCLEAR(char, uint8_t)
601 CSAN_ATOMIC_FUNC_TESTANDSET(char, uint8_t)
602 #endif
603
604 CSAN_ATOMIC_FUNC_ADD(short, uint16_t)
605 CSAN_ATOMIC_FUNC_CLEAR(short, uint16_t)
606 CSAN_ATOMIC_FUNC_CMPSET(short, uint16_t)
607 CSAN_ATOMIC_FUNC_FCMPSET(short, uint16_t)
608 CSAN_ATOMIC_FUNC_LOAD(short, uint16_t)
609 CSAN_ATOMIC_FUNC_SET(short, uint16_t)
610 CSAN_ATOMIC_FUNC_SUBTRACT(short, uint16_t)
611 _CSAN_ATOMIC_FUNC_STORE(short, uint16_t)
612 #if 0
613 CSAN_ATOMIC_FUNC_FETCHADD(short, uint16_t)
614 CSAN_ATOMIC_FUNC_READANDCLEAR(short, uint16_t)
615 CSAN_ATOMIC_FUNC_SWAP(short, uint16_t)
616 CSAN_ATOMIC_FUNC_TESTANDCLEAR(short, uint16_t)
617 CSAN_ATOMIC_FUNC_TESTANDSET(short, uint16_t)
618 #endif
619
620 CSAN_ATOMIC_FUNC_ADD(int, u_int)
621 CSAN_ATOMIC_FUNC_CLEAR(int, u_int)
622 CSAN_ATOMIC_FUNC_CMPSET(int, u_int)
623 CSAN_ATOMIC_FUNC_FCMPSET(int, u_int)
624 CSAN_ATOMIC_FUNC_FETCHADD(int, u_int)
625 CSAN_ATOMIC_FUNC_LOAD(int, u_int)
626 CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int)
627 CSAN_ATOMIC_FUNC_SET(int, u_int)
628 CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int)
629 CSAN_ATOMIC_FUNC_STORE(int, u_int)
630 CSAN_ATOMIC_FUNC_SWAP(int, u_int)
631 CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int)
632 CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int)
633
634 CSAN_ATOMIC_FUNC_ADD(long, u_long)
635 CSAN_ATOMIC_FUNC_CLEAR(long, u_long)
636 CSAN_ATOMIC_FUNC_CMPSET(long, u_long)
637 CSAN_ATOMIC_FUNC_FCMPSET(long, u_long)
638 CSAN_ATOMIC_FUNC_FETCHADD(long, u_long)
639 CSAN_ATOMIC_FUNC_LOAD(long, u_long)
640 CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long)
641 CSAN_ATOMIC_FUNC_SET(long, u_long)
642 CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long)
643 CSAN_ATOMIC_FUNC_STORE(long, u_long)
644 CSAN_ATOMIC_FUNC_SWAP(long, u_long)
645 CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long)
646 CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long)
647 CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long)
648
649 CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t)
650 CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t)
651 CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t)
652 CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t)
653 #if !defined(__amd64__)
654 CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t)
655 #endif
656 CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t)
657 CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t)
658 CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t)
659 CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t)
660 CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t)
661 CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t)
662 #if 0
663 CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t)
664 CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t)
665 #endif
666
667 #define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \
668 void kcsan_atomic_thread_fence_##name(void) \
669 { \
670 atomic_thread_fence_##name(); \
671 }
672
673 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq)
674 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel)
675 CSAN_ATOMIC_FUNC_THREAD_FENCE(rel)
676 CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst)
677
678 void
679 kcsan_atomic_interrupt_fence(void)
680 {
681 atomic_interrupt_fence();
682 }
683
684 /* -------------------------------------------------------------------------- */
685
686 #include <sys/bus.h>
687 #include <machine/bus.h>
688 #include <sys/bus_san.h>
689
690 int
kcsan_bus_space_map(bus_space_tag_t tag,bus_addr_t hnd,bus_size_t size,int flags,bus_space_handle_t * handlep)691 kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
692 int flags, bus_space_handle_t *handlep)
693 {
694
695 return (bus_space_map(tag, hnd, size, flags, handlep));
696 }
697
698 void
kcsan_bus_space_unmap(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t size)699 kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
700 bus_size_t size)
701 {
702
703 bus_space_unmap(tag, hnd, size);
704 }
705
706 int
kcsan_bus_space_subregion(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t offset,bus_size_t size,bus_space_handle_t * handlep)707 kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
708 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
709 {
710
711 return (bus_space_subregion(tag, hnd, offset, size, handlep));
712 }
713
714 #if !defined(__amd64__)
715 int
kcsan_bus_space_alloc(bus_space_tag_t tag,bus_addr_t reg_start,bus_addr_t reg_end,bus_size_t size,bus_size_t alignment,bus_size_t boundary,int flags,bus_addr_t * addrp,bus_space_handle_t * handlep)716 kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start,
717 bus_addr_t reg_end, bus_size_t size, bus_size_t alignment,
718 bus_size_t boundary, int flags, bus_addr_t *addrp,
719 bus_space_handle_t *handlep)
720 {
721
722 return (bus_space_alloc(tag, reg_start, reg_end, size, alignment,
723 boundary, flags, addrp, handlep));
724 }
725 #endif
726
727 void
kcsan_bus_space_free(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t size)728 kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
729 bus_size_t size)
730 {
731
732 bus_space_free(tag, hnd, size);
733 }
734
735 void
kcsan_bus_space_barrier(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t offset,bus_size_t size,int flags)736 kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
737 bus_size_t offset, bus_size_t size, int flags)
738 {
739
740 bus_space_barrier(tag, hnd, offset, size, flags);
741 }
742
743 #define CSAN_BUS_READ_FUNC(func, width, type) \
744 type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \
745 bus_space_handle_t hnd, bus_size_t offset) \
746 { \
747 return (bus_space_read##func##_##width(tag, hnd, \
748 offset)); \
749 } \
750
751 #define CSAN_BUS_READ_PTR_FUNC(func, width, type) \
752 void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
753 bus_space_handle_t hnd, bus_size_t size, type *buf, \
754 bus_size_t count) \
755 { \
756 kcsan_access((uintptr_t)buf, sizeof(type) * count, \
757 false, false, __RET_ADDR); \
758 bus_space_read_##func##_##width(tag, hnd, size, buf, \
759 count); \
760 }
761
762 CSAN_BUS_READ_FUNC(, 1, uint8_t)
763 CSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
764 CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
765 CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
766 CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
767 CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
768
769 CSAN_BUS_READ_FUNC(, 2, uint16_t)
770 CSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
771 CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
772 CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
773 CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
774 CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
775
776 CSAN_BUS_READ_FUNC(, 4, uint32_t)
777 CSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
778 CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
779 CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
780 CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
781 CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
782
783 CSAN_BUS_READ_FUNC(, 8, uint64_t)
784 #if defined(__aarch64__)
785 CSAN_BUS_READ_FUNC(_stream, 8, uint64_t)
786 CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
787 CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
788 CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
789 CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
790 #endif
791
792 #define CSAN_BUS_WRITE_FUNC(func, width, type) \
793 void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
794 bus_space_handle_t hnd, bus_size_t offset, type value) \
795 { \
796 bus_space_write##func##_##width(tag, hnd, offset, value); \
797 } \
798
799 #define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \
800 void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \
801 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
802 bus_size_t count) \
803 { \
804 kcsan_access((uintptr_t)buf, sizeof(type) * count, \
805 true, false, __RET_ADDR); \
806 bus_space_write_##func##_##width(tag, hnd, size, buf, \
807 count); \
808 }
809
810 CSAN_BUS_WRITE_FUNC(, 1, uint8_t)
811 CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
812 CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
813 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
814 CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
815 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
816
817 CSAN_BUS_WRITE_FUNC(, 2, uint16_t)
818 CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
819 CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
820 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
821 CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
822 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
823
824 CSAN_BUS_WRITE_FUNC(, 4, uint32_t)
825 CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
826 CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
827 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
828 CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
829 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
830
831 CSAN_BUS_WRITE_FUNC(, 8, uint64_t)
832 #if defined(__aarch64__)
833 CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t)
834 CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t)
835 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t)
836 CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t)
837 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t)
838 #endif
839
840 #define CSAN_BUS_SET_FUNC(func, width, type) \
841 void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
842 bus_space_handle_t hnd, bus_size_t offset, type value, \
843 bus_size_t count) \
844 { \
845 bus_space_set_##func##_##width(tag, hnd, offset, value, \
846 count); \
847 }
848
849 CSAN_BUS_SET_FUNC(multi, 1, uint8_t)
850 CSAN_BUS_SET_FUNC(region, 1, uint8_t)
851 #if !defined(__aarch64__)
852 CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
853 CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
854 #endif
855
856 CSAN_BUS_SET_FUNC(multi, 2, uint16_t)
857 CSAN_BUS_SET_FUNC(region, 2, uint16_t)
858 #if !defined(__aarch64__)
859 CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
860 CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
861 #endif
862
863 CSAN_BUS_SET_FUNC(multi, 4, uint32_t)
864 CSAN_BUS_SET_FUNC(region, 4, uint32_t)
865 #if !defined(__aarch64__)
866 CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
867 CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
868 #endif
869
870 #if !defined(__amd64__)
871 CSAN_BUS_SET_FUNC(multi, 8, uint64_t)
872 CSAN_BUS_SET_FUNC(region, 8, uint64_t)
873 #if !defined(__aarch64__)
874 CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t)
875 CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t)
876 #endif
877 #endif
878
879 #define CSAN_BUS_PEEK_FUNC(width, type) \
880 int kcsan_bus_space_peek_##width(bus_space_tag_t tag, \
881 bus_space_handle_t hnd, bus_size_t offset, type *value) \
882 { \
883 kcsan_access((uintptr_t)value, sizeof(type), true, false, \
884 __RET_ADDR); \
885 return (bus_space_peek_##width(tag, hnd, offset, value)); \
886 }
887
888 CSAN_BUS_PEEK_FUNC(1, uint8_t)
889 CSAN_BUS_PEEK_FUNC(2, uint16_t)
890 CSAN_BUS_PEEK_FUNC(4, uint32_t)
891 #if !defined(__i386__)
892 CSAN_BUS_PEEK_FUNC(8, uint64_t)
893 #endif
894
895 #define CSAN_BUS_POKE_FUNC(width, type) \
896 int kcsan_bus_space_poke_##width(bus_space_tag_t tag, \
897 bus_space_handle_t hnd, bus_size_t offset, type value) \
898 { \
899 return (bus_space_poke_##width(tag, hnd, offset, value)); \
900 }
901
902 CSAN_BUS_POKE_FUNC(1, uint8_t)
903 CSAN_BUS_POKE_FUNC(2, uint16_t)
904 CSAN_BUS_POKE_FUNC(4, uint32_t)
905 #if !defined(__i386__)
906 CSAN_BUS_POKE_FUNC(8, uint64_t)
907 #endif
908