1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KCSAN test with various race scenarious to test runtime behaviour. Since the 4 * interface with which KCSAN's reports are obtained is via the console, this is 5 * the output we should verify. For each test case checks the presence (or 6 * absence) of generated reports. Relies on 'console' tracepoint to capture 7 * reports as they appear in the kernel log. 8 * 9 * Makes use of KUnit for test organization, and the Torture framework for test 10 * thread control. 11 * 12 * Copyright (C) 2020, Google LLC. 13 * Author: Marco Elver <[email protected]> 14 */ 15 16 #define pr_fmt(fmt) "kcsan_test: " fmt 17 18 #include <kunit/test.h> 19 #include <linux/atomic.h> 20 #include <linux/bitops.h> 21 #include <linux/jiffies.h> 22 #include <linux/kcsan-checks.h> 23 #include <linux/kernel.h> 24 #include <linux/mutex.h> 25 #include <linux/sched.h> 26 #include <linux/seqlock.h> 27 #include <linux/spinlock.h> 28 #include <linux/string.h> 29 #include <linux/timer.h> 30 #include <linux/torture.h> 31 #include <linux/tracepoint.h> 32 #include <linux/types.h> 33 #include <trace/events/printk.h> 34 35 #define KCSAN_TEST_REQUIRES(test, cond) do { \ 36 if (!(cond)) \ 37 kunit_skip((test), "Test requires: " #cond); \ 38 } while (0) 39 40 #ifdef CONFIG_CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE 41 #define __KCSAN_ACCESS_RW(alt) (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE) 42 #else 43 #define __KCSAN_ACCESS_RW(alt) (alt) 44 #endif 45 46 /* Points to current test-case memory access "kernels". */ 47 static void (*access_kernels[2])(void); 48 49 static struct task_struct **threads; /* Lists of threads. */ 50 static unsigned long end_time; /* End time of test. */ 51 52 /* Report as observed from console. */ 53 static struct { 54 spinlock_t lock; 55 int nlines; 56 char lines[3][512]; 57 } observed = { 58 .lock = __SPIN_LOCK_UNLOCKED(observed.lock), 59 }; 60 61 /* Setup test checking loop. */ 62 static __no_kcsan inline void 63 begin_test_checks(void (*func1)(void), void (*func2)(void)) 64 { 65 kcsan_disable_current(); 66 67 /* 68 * Require at least as long as KCSAN_REPORT_ONCE_IN_MS, to ensure at 69 * least one race is reported. 70 */ 71 end_time = jiffies + msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS + 500); 72 73 /* Signal start; release potential initialization of shared data. */ 74 smp_store_release(&access_kernels[0], func1); 75 smp_store_release(&access_kernels[1], func2); 76 } 77 78 /* End test checking loop. */ 79 static __no_kcsan inline bool 80 end_test_checks(bool stop) 81 { 82 if (!stop && time_before(jiffies, end_time)) { 83 /* Continue checking */ 84 might_sleep(); 85 return false; 86 } 87 88 kcsan_enable_current(); 89 return true; 90 } 91 92 /* 93 * Probe for console output: checks if a race was reported, and obtains observed 94 * lines of interest. 95 */ 96 __no_kcsan 97 static void probe_console(void *ignore, const char *buf, size_t len) 98 { 99 unsigned long flags; 100 int nlines; 101 102 /* 103 * Note that KCSAN reports under a global lock, so we do not risk the 104 * possibility of having multiple reports interleaved. If that were the 105 * case, we'd expect tests to fail. 106 */ 107 108 spin_lock_irqsave(&observed.lock, flags); 109 nlines = observed.nlines; 110 111 if (strnstr(buf, "BUG: KCSAN: ", len) && strnstr(buf, "test_", len)) { 112 /* 113 * KCSAN report and related to the test. 114 * 115 * The provided @buf is not NUL-terminated; copy no more than 116 * @len bytes and let strscpy() add the missing NUL-terminator. 117 */ 118 strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0]))); 119 nlines = 1; 120 } else if ((nlines == 1 || nlines == 2) && strnstr(buf, "bytes by", len)) { 121 strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0]))); 122 123 if (strnstr(buf, "race at unknown origin", len)) { 124 if (WARN_ON(nlines != 2)) 125 goto out; 126 127 /* No second line of interest. */ 128 strcpy(observed.lines[nlines++], "<none>"); 129 } 130 } 131 132 out: 133 WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */ 134 spin_unlock_irqrestore(&observed.lock, flags); 135 } 136 137 /* Check if a report related to the test exists. */ 138 __no_kcsan 139 static bool report_available(void) 140 { 141 return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines); 142 } 143 144 /* Report information we expect in a report. */ 145 struct expect_report { 146 /* Access information of both accesses. */ 147 struct { 148 void *fn; /* Function pointer to expected function of top frame. */ 149 void *addr; /* Address of access; unchecked if NULL. */ 150 size_t size; /* Size of access; unchecked if @addr is NULL. */ 151 int type; /* Access type, see KCSAN_ACCESS definitions. */ 152 } access[2]; 153 }; 154 155 /* Check observed report matches information in @r. */ 156 __no_kcsan 157 static bool __report_matches(const struct expect_report *r) 158 { 159 const bool is_assert = (r->access[0].type | r->access[1].type) & KCSAN_ACCESS_ASSERT; 160 bool ret = false; 161 unsigned long flags; 162 typeof(observed.lines) expect; 163 const char *end; 164 char *cur; 165 int i; 166 167 /* Doubled-checked locking. */ 168 if (!report_available()) 169 return false; 170 171 /* Generate expected report contents. */ 172 173 /* Title */ 174 cur = expect[0]; 175 end = &expect[0][sizeof(expect[0]) - 1]; 176 cur += scnprintf(cur, end - cur, "BUG: KCSAN: %s in ", 177 is_assert ? "assert: race" : "data-race"); 178 if (r->access[1].fn) { 179 char tmp[2][64]; 180 int cmp; 181 182 /* Expect lexographically sorted function names in title. */ 183 scnprintf(tmp[0], sizeof(tmp[0]), "%pS", r->access[0].fn); 184 scnprintf(tmp[1], sizeof(tmp[1]), "%pS", r->access[1].fn); 185 cmp = strcmp(tmp[0], tmp[1]); 186 cur += scnprintf(cur, end - cur, "%ps / %ps", 187 cmp < 0 ? r->access[0].fn : r->access[1].fn, 188 cmp < 0 ? r->access[1].fn : r->access[0].fn); 189 } else { 190 scnprintf(cur, end - cur, "%pS", r->access[0].fn); 191 /* The exact offset won't match, remove it. */ 192 cur = strchr(expect[0], '+'); 193 if (cur) 194 *cur = '\0'; 195 } 196 197 /* Access 1 */ 198 cur = expect[1]; 199 end = &expect[1][sizeof(expect[1]) - 1]; 200 if (!r->access[1].fn) 201 cur += scnprintf(cur, end - cur, "race at unknown origin, with "); 202 203 /* Access 1 & 2 */ 204 for (i = 0; i < 2; ++i) { 205 const int ty = r->access[i].type; 206 const char *const access_type = 207 (ty & KCSAN_ACCESS_ASSERT) ? 208 ((ty & KCSAN_ACCESS_WRITE) ? 209 "assert no accesses" : 210 "assert no writes") : 211 ((ty & KCSAN_ACCESS_WRITE) ? 212 ((ty & KCSAN_ACCESS_COMPOUND) ? 213 "read-write" : 214 "write") : 215 "read"); 216 const bool is_atomic = (ty & KCSAN_ACCESS_ATOMIC); 217 const bool is_scoped = (ty & KCSAN_ACCESS_SCOPED); 218 const char *const access_type_aux = 219 (is_atomic && is_scoped) ? " (marked, reordered)" 220 : (is_atomic ? " (marked)" 221 : (is_scoped ? " (reordered)" : "")); 222 223 if (i == 1) { 224 /* Access 2 */ 225 cur = expect[2]; 226 end = &expect[2][sizeof(expect[2]) - 1]; 227 228 if (!r->access[1].fn) { 229 /* Dummy string if no second access is available. */ 230 strcpy(cur, "<none>"); 231 break; 232 } 233 } 234 235 cur += scnprintf(cur, end - cur, "%s%s to ", access_type, 236 access_type_aux); 237 238 if (r->access[i].addr) /* Address is optional. */ 239 cur += scnprintf(cur, end - cur, "0x%px of %zu bytes", 240 r->access[i].addr, r->access[i].size); 241 } 242 243 spin_lock_irqsave(&observed.lock, flags); 244 if (!report_available()) 245 goto out; /* A new report is being captured. */ 246 247 /* Finally match expected output to what we actually observed. */ 248 ret = strstr(observed.lines[0], expect[0]) && 249 /* Access info may appear in any order. */ 250 ((strstr(observed.lines[1], expect[1]) && 251 strstr(observed.lines[2], expect[2])) || 252 (strstr(observed.lines[1], expect[2]) && 253 strstr(observed.lines[2], expect[1]))); 254 out: 255 spin_unlock_irqrestore(&observed.lock, flags); 256 return ret; 257 } 258 259 static __always_inline const struct expect_report * 260 __report_set_scoped(struct expect_report *r, int accesses) 261 { 262 BUILD_BUG_ON(accesses > 3); 263 264 if (accesses & 1) 265 r->access[0].type |= KCSAN_ACCESS_SCOPED; 266 else 267 r->access[0].type &= ~KCSAN_ACCESS_SCOPED; 268 269 if (accesses & 2) 270 r->access[1].type |= KCSAN_ACCESS_SCOPED; 271 else 272 r->access[1].type &= ~KCSAN_ACCESS_SCOPED; 273 274 return r; 275 } 276 277 __no_kcsan 278 static bool report_matches_any_reordered(struct expect_report *r) 279 { 280 return __report_matches(__report_set_scoped(r, 0)) || 281 __report_matches(__report_set_scoped(r, 1)) || 282 __report_matches(__report_set_scoped(r, 2)) || 283 __report_matches(__report_set_scoped(r, 3)); 284 } 285 286 #ifdef CONFIG_KCSAN_WEAK_MEMORY 287 /* Due to reordering accesses, any access may appear as "(reordered)". */ 288 #define report_matches report_matches_any_reordered 289 #else 290 #define report_matches __report_matches 291 #endif 292 293 /* ===== Test kernels ===== */ 294 295 static long test_sink; 296 static long test_var; 297 /* @test_array should be large enough to fall into multiple watchpoint slots. */ 298 static long test_array[3 * PAGE_SIZE / sizeof(long)]; 299 static struct { 300 long val[8]; 301 } test_struct; 302 static DEFINE_SEQLOCK(test_seqlock); 303 304 /* 305 * Helper to avoid compiler optimizing out reads, and to generate source values 306 * for writes. 307 */ 308 __no_kcsan 309 static noinline void sink_value(long v) { WRITE_ONCE(test_sink, v); } 310 311 /* 312 * Generates a delay and some accesses that enter the runtime but do not produce 313 * data races. 314 */ 315 static noinline void test_delay(int iter) 316 { 317 while (iter--) 318 sink_value(READ_ONCE(test_sink)); 319 } 320 321 static noinline void test_kernel_read(void) { sink_value(test_var); } 322 323 static noinline void test_kernel_write(void) 324 { 325 test_var = READ_ONCE_NOCHECK(test_sink) + 1; 326 } 327 328 static noinline void test_kernel_write_nochange(void) { test_var = 42; } 329 330 /* Suffixed by value-change exception filter. */ 331 static noinline void test_kernel_write_nochange_rcu(void) { test_var = 42; } 332 333 static noinline void test_kernel_read_atomic(void) 334 { 335 sink_value(READ_ONCE(test_var)); 336 } 337 338 static noinline void test_kernel_write_atomic(void) 339 { 340 WRITE_ONCE(test_var, READ_ONCE_NOCHECK(test_sink) + 1); 341 } 342 343 static noinline void test_kernel_atomic_rmw(void) 344 { 345 /* Use builtin, so we can set up the "bad" atomic/non-atomic scenario. */ 346 __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED); 347 } 348 349 __no_kcsan 350 static noinline void test_kernel_write_uninstrumented(void) { test_var++; } 351 352 static noinline void test_kernel_data_race(void) { data_race(test_var++); } 353 354 static noinline void test_kernel_assert_writer(void) 355 { 356 ASSERT_EXCLUSIVE_WRITER(test_var); 357 } 358 359 static noinline void test_kernel_assert_access(void) 360 { 361 ASSERT_EXCLUSIVE_ACCESS(test_var); 362 } 363 364 #define TEST_CHANGE_BITS 0xff00ff00 365 366 static noinline void test_kernel_change_bits(void) 367 { 368 if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { 369 /* 370 * Avoid race of unknown origin for this test, just pretend they 371 * are atomic. 372 */ 373 kcsan_nestable_atomic_begin(); 374 test_var ^= TEST_CHANGE_BITS; 375 kcsan_nestable_atomic_end(); 376 } else 377 WRITE_ONCE(test_var, READ_ONCE(test_var) ^ TEST_CHANGE_BITS); 378 } 379 380 static noinline void test_kernel_assert_bits_change(void) 381 { 382 ASSERT_EXCLUSIVE_BITS(test_var, TEST_CHANGE_BITS); 383 } 384 385 static noinline void test_kernel_assert_bits_nochange(void) 386 { 387 ASSERT_EXCLUSIVE_BITS(test_var, ~TEST_CHANGE_BITS); 388 } 389 390 /* 391 * Scoped assertions do trigger anywhere in scope. However, the report should 392 * still only point at the start of the scope. 393 */ 394 static noinline void test_enter_scope(void) 395 { 396 int x = 0; 397 398 /* Unrelated accesses to scoped assert. */ 399 READ_ONCE(test_sink); 400 kcsan_check_read(&x, sizeof(x)); 401 } 402 403 static noinline void test_kernel_assert_writer_scoped(void) 404 { 405 ASSERT_EXCLUSIVE_WRITER_SCOPED(test_var); 406 test_enter_scope(); 407 } 408 409 static noinline void test_kernel_assert_access_scoped(void) 410 { 411 ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_var); 412 test_enter_scope(); 413 } 414 415 static noinline void test_kernel_rmw_array(void) 416 { 417 int i; 418 419 for (i = 0; i < ARRAY_SIZE(test_array); ++i) 420 test_array[i]++; 421 } 422 423 static noinline void test_kernel_write_struct(void) 424 { 425 kcsan_check_write(&test_struct, sizeof(test_struct)); 426 kcsan_disable_current(); 427 test_struct.val[3]++; /* induce value change */ 428 kcsan_enable_current(); 429 } 430 431 static noinline void test_kernel_write_struct_part(void) 432 { 433 test_struct.val[3] = 42; 434 } 435 436 static noinline void test_kernel_read_struct_zero_size(void) 437 { 438 kcsan_check_read(&test_struct.val[3], 0); 439 } 440 441 static noinline void test_kernel_jiffies_reader(void) 442 { 443 sink_value((long)jiffies); 444 } 445 446 static noinline void test_kernel_seqlock_reader(void) 447 { 448 unsigned int seq; 449 450 do { 451 seq = read_seqbegin(&test_seqlock); 452 sink_value(test_var); 453 } while (read_seqretry(&test_seqlock, seq)); 454 } 455 456 static noinline void test_kernel_seqlock_writer(void) 457 { 458 unsigned long flags; 459 460 write_seqlock_irqsave(&test_seqlock, flags); 461 test_var++; 462 write_sequnlock_irqrestore(&test_seqlock, flags); 463 } 464 465 static noinline void test_kernel_atomic_builtins(void) 466 { 467 /* 468 * Generate concurrent accesses, expecting no reports, ensuring KCSAN 469 * treats builtin atomics as actually atomic. 470 */ 471 __atomic_load_n(&test_var, __ATOMIC_RELAXED); 472 } 473 474 static noinline void test_kernel_xor_1bit(void) 475 { 476 /* Do not report data races between the read-writes. */ 477 kcsan_nestable_atomic_begin(); 478 test_var ^= 0x10000; 479 kcsan_nestable_atomic_end(); 480 } 481 482 #define TEST_KERNEL_LOCKED(name, acquire, release) \ 483 static noinline void test_kernel_##name(void) \ 484 { \ 485 long *flag = &test_struct.val[0]; \ 486 long v = 0; \ 487 if (!(acquire)) \ 488 return; \ 489 while (v++ < 100) { \ 490 test_var++; \ 491 barrier(); \ 492 } \ 493 release; \ 494 test_delay(10); \ 495 } 496 497 TEST_KERNEL_LOCKED(with_memorder, 498 cmpxchg_acquire(flag, 0, 1) == 0, 499 smp_store_release(flag, 0)); 500 TEST_KERNEL_LOCKED(wrong_memorder, 501 cmpxchg_relaxed(flag, 0, 1) == 0, 502 WRITE_ONCE(*flag, 0)); 503 TEST_KERNEL_LOCKED(atomic_builtin_with_memorder, 504 __atomic_compare_exchange_n(flag, &v, 1, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED), 505 __atomic_store_n(flag, 0, __ATOMIC_RELEASE)); 506 TEST_KERNEL_LOCKED(atomic_builtin_wrong_memorder, 507 __atomic_compare_exchange_n(flag, &v, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED), 508 __atomic_store_n(flag, 0, __ATOMIC_RELAXED)); 509 510 /* ===== Test cases ===== */ 511 512 /* 513 * Tests that various barriers have the expected effect on internal state. Not 514 * exhaustive on atomic_t operations. Unlike the selftest, also checks for 515 * too-strict barrier instrumentation; these can be tolerated, because it does 516 * not cause false positives, but at least we should be aware of such cases. 517 */ 518 static void test_barrier_nothreads(struct kunit *test) 519 { 520 #ifdef CONFIG_KCSAN_WEAK_MEMORY 521 struct kcsan_scoped_access *reorder_access = ¤t->kcsan_ctx.reorder_access; 522 #else 523 struct kcsan_scoped_access *reorder_access = NULL; 524 #endif 525 arch_spinlock_t arch_spinlock = __ARCH_SPIN_LOCK_UNLOCKED; 526 DEFINE_SPINLOCK(spinlock); 527 DEFINE_MUTEX(mutex); 528 atomic_t dummy; 529 530 KCSAN_TEST_REQUIRES(test, reorder_access != NULL); 531 KCSAN_TEST_REQUIRES(test, IS_ENABLED(CONFIG_SMP)); 532 533 #define __KCSAN_EXPECT_BARRIER(access_type, barrier, order_before, name) \ 534 do { \ 535 reorder_access->type = (access_type) | KCSAN_ACCESS_SCOPED; \ 536 reorder_access->size = sizeof(test_var); \ 537 barrier; \ 538 KUNIT_EXPECT_EQ_MSG(test, reorder_access->size, \ 539 order_before ? 0 : sizeof(test_var), \ 540 "improperly instrumented type=(" #access_type "): " name); \ 541 } while (0) 542 #define KCSAN_EXPECT_READ_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(0, b, o, #b) 543 #define KCSAN_EXPECT_WRITE_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(KCSAN_ACCESS_WRITE, b, o, #b) 544 #define KCSAN_EXPECT_RW_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE, b, o, #b) 545 546 /* Force creating a valid entry in reorder_access first. */ 547 test_var = 0; 548 while (test_var++ < 1000000 && reorder_access->size != sizeof(test_var)) 549 __kcsan_check_read(&test_var, sizeof(test_var)); 550 KUNIT_ASSERT_EQ(test, reorder_access->size, sizeof(test_var)); 551 552 kcsan_nestable_atomic_begin(); /* No watchpoints in called functions. */ 553 554 KCSAN_EXPECT_READ_BARRIER(mb(), true); 555 KCSAN_EXPECT_READ_BARRIER(wmb(), false); 556 KCSAN_EXPECT_READ_BARRIER(rmb(), true); 557 KCSAN_EXPECT_READ_BARRIER(smp_mb(), true); 558 KCSAN_EXPECT_READ_BARRIER(smp_wmb(), false); 559 KCSAN_EXPECT_READ_BARRIER(smp_rmb(), true); 560 KCSAN_EXPECT_READ_BARRIER(dma_wmb(), false); 561 KCSAN_EXPECT_READ_BARRIER(dma_rmb(), true); 562 KCSAN_EXPECT_READ_BARRIER(smp_mb__before_atomic(), true); 563 KCSAN_EXPECT_READ_BARRIER(smp_mb__after_atomic(), true); 564 KCSAN_EXPECT_READ_BARRIER(smp_mb__after_spinlock(), true); 565 KCSAN_EXPECT_READ_BARRIER(smp_store_mb(test_var, 0), true); 566 KCSAN_EXPECT_READ_BARRIER(smp_load_acquire(&test_var), false); 567 KCSAN_EXPECT_READ_BARRIER(smp_store_release(&test_var, 0), true); 568 KCSAN_EXPECT_READ_BARRIER(xchg(&test_var, 0), true); 569 KCSAN_EXPECT_READ_BARRIER(xchg_release(&test_var, 0), true); 570 KCSAN_EXPECT_READ_BARRIER(xchg_relaxed(&test_var, 0), false); 571 KCSAN_EXPECT_READ_BARRIER(cmpxchg(&test_var, 0, 0), true); 572 KCSAN_EXPECT_READ_BARRIER(cmpxchg_release(&test_var, 0, 0), true); 573 KCSAN_EXPECT_READ_BARRIER(cmpxchg_relaxed(&test_var, 0, 0), false); 574 KCSAN_EXPECT_READ_BARRIER(atomic_read(&dummy), false); 575 KCSAN_EXPECT_READ_BARRIER(atomic_read_acquire(&dummy), false); 576 KCSAN_EXPECT_READ_BARRIER(atomic_set(&dummy, 0), false); 577 KCSAN_EXPECT_READ_BARRIER(atomic_set_release(&dummy, 0), true); 578 KCSAN_EXPECT_READ_BARRIER(atomic_add(1, &dummy), false); 579 KCSAN_EXPECT_READ_BARRIER(atomic_add_return(1, &dummy), true); 580 KCSAN_EXPECT_READ_BARRIER(atomic_add_return_acquire(1, &dummy), false); 581 KCSAN_EXPECT_READ_BARRIER(atomic_add_return_release(1, &dummy), true); 582 KCSAN_EXPECT_READ_BARRIER(atomic_add_return_relaxed(1, &dummy), false); 583 KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add(1, &dummy), true); 584 KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_acquire(1, &dummy), false); 585 KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_release(1, &dummy), true); 586 KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false); 587 KCSAN_EXPECT_READ_BARRIER(test_and_set_bit(0, &test_var), true); 588 KCSAN_EXPECT_READ_BARRIER(test_and_clear_bit(0, &test_var), true); 589 KCSAN_EXPECT_READ_BARRIER(test_and_change_bit(0, &test_var), true); 590 KCSAN_EXPECT_READ_BARRIER(clear_bit_unlock(0, &test_var), true); 591 KCSAN_EXPECT_READ_BARRIER(__clear_bit_unlock(0, &test_var), true); 592 KCSAN_EXPECT_READ_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var), true); 593 KCSAN_EXPECT_READ_BARRIER(arch_spin_lock(&arch_spinlock), false); 594 KCSAN_EXPECT_READ_BARRIER(arch_spin_unlock(&arch_spinlock), true); 595 KCSAN_EXPECT_READ_BARRIER(spin_lock(&spinlock), false); 596 KCSAN_EXPECT_READ_BARRIER(spin_unlock(&spinlock), true); 597 KCSAN_EXPECT_READ_BARRIER(mutex_lock(&mutex), false); 598 KCSAN_EXPECT_READ_BARRIER(mutex_unlock(&mutex), true); 599 600 KCSAN_EXPECT_WRITE_BARRIER(mb(), true); 601 KCSAN_EXPECT_WRITE_BARRIER(wmb(), true); 602 KCSAN_EXPECT_WRITE_BARRIER(rmb(), false); 603 KCSAN_EXPECT_WRITE_BARRIER(smp_mb(), true); 604 KCSAN_EXPECT_WRITE_BARRIER(smp_wmb(), true); 605 KCSAN_EXPECT_WRITE_BARRIER(smp_rmb(), false); 606 KCSAN_EXPECT_WRITE_BARRIER(dma_wmb(), true); 607 KCSAN_EXPECT_WRITE_BARRIER(dma_rmb(), false); 608 KCSAN_EXPECT_WRITE_BARRIER(smp_mb__before_atomic(), true); 609 KCSAN_EXPECT_WRITE_BARRIER(smp_mb__after_atomic(), true); 610 KCSAN_EXPECT_WRITE_BARRIER(smp_mb__after_spinlock(), true); 611 KCSAN_EXPECT_WRITE_BARRIER(smp_store_mb(test_var, 0), true); 612 KCSAN_EXPECT_WRITE_BARRIER(smp_load_acquire(&test_var), false); 613 KCSAN_EXPECT_WRITE_BARRIER(smp_store_release(&test_var, 0), true); 614 KCSAN_EXPECT_WRITE_BARRIER(xchg(&test_var, 0), true); 615 KCSAN_EXPECT_WRITE_BARRIER(xchg_release(&test_var, 0), true); 616 KCSAN_EXPECT_WRITE_BARRIER(xchg_relaxed(&test_var, 0), false); 617 KCSAN_EXPECT_WRITE_BARRIER(cmpxchg(&test_var, 0, 0), true); 618 KCSAN_EXPECT_WRITE_BARRIER(cmpxchg_release(&test_var, 0, 0), true); 619 KCSAN_EXPECT_WRITE_BARRIER(cmpxchg_relaxed(&test_var, 0, 0), false); 620 KCSAN_EXPECT_WRITE_BARRIER(atomic_read(&dummy), false); 621 KCSAN_EXPECT_WRITE_BARRIER(atomic_read_acquire(&dummy), false); 622 KCSAN_EXPECT_WRITE_BARRIER(atomic_set(&dummy, 0), false); 623 KCSAN_EXPECT_WRITE_BARRIER(atomic_set_release(&dummy, 0), true); 624 KCSAN_EXPECT_WRITE_BARRIER(atomic_add(1, &dummy), false); 625 KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return(1, &dummy), true); 626 KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_acquire(1, &dummy), false); 627 KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_release(1, &dummy), true); 628 KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_relaxed(1, &dummy), false); 629 KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add(1, &dummy), true); 630 KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_acquire(1, &dummy), false); 631 KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_release(1, &dummy), true); 632 KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false); 633 KCSAN_EXPECT_WRITE_BARRIER(test_and_set_bit(0, &test_var), true); 634 KCSAN_EXPECT_WRITE_BARRIER(test_and_clear_bit(0, &test_var), true); 635 KCSAN_EXPECT_WRITE_BARRIER(test_and_change_bit(0, &test_var), true); 636 KCSAN_EXPECT_WRITE_BARRIER(clear_bit_unlock(0, &test_var), true); 637 KCSAN_EXPECT_WRITE_BARRIER(__clear_bit_unlock(0, &test_var), true); 638 KCSAN_EXPECT_WRITE_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var), true); 639 KCSAN_EXPECT_WRITE_BARRIER(arch_spin_lock(&arch_spinlock), false); 640 KCSAN_EXPECT_WRITE_BARRIER(arch_spin_unlock(&arch_spinlock), true); 641 KCSAN_EXPECT_WRITE_BARRIER(spin_lock(&spinlock), false); 642 KCSAN_EXPECT_WRITE_BARRIER(spin_unlock(&spinlock), true); 643 KCSAN_EXPECT_WRITE_BARRIER(mutex_lock(&mutex), false); 644 KCSAN_EXPECT_WRITE_BARRIER(mutex_unlock(&mutex), true); 645 646 KCSAN_EXPECT_RW_BARRIER(mb(), true); 647 KCSAN_EXPECT_RW_BARRIER(wmb(), true); 648 KCSAN_EXPECT_RW_BARRIER(rmb(), true); 649 KCSAN_EXPECT_RW_BARRIER(smp_mb(), true); 650 KCSAN_EXPECT_RW_BARRIER(smp_wmb(), true); 651 KCSAN_EXPECT_RW_BARRIER(smp_rmb(), true); 652 KCSAN_EXPECT_RW_BARRIER(dma_wmb(), true); 653 KCSAN_EXPECT_RW_BARRIER(dma_rmb(), true); 654 KCSAN_EXPECT_RW_BARRIER(smp_mb__before_atomic(), true); 655 KCSAN_EXPECT_RW_BARRIER(smp_mb__after_atomic(), true); 656 KCSAN_EXPECT_RW_BARRIER(smp_mb__after_spinlock(), true); 657 KCSAN_EXPECT_RW_BARRIER(smp_store_mb(test_var, 0), true); 658 KCSAN_EXPECT_RW_BARRIER(smp_load_acquire(&test_var), false); 659 KCSAN_EXPECT_RW_BARRIER(smp_store_release(&test_var, 0), true); 660 KCSAN_EXPECT_RW_BARRIER(xchg(&test_var, 0), true); 661 KCSAN_EXPECT_RW_BARRIER(xchg_release(&test_var, 0), true); 662 KCSAN_EXPECT_RW_BARRIER(xchg_relaxed(&test_var, 0), false); 663 KCSAN_EXPECT_RW_BARRIER(cmpxchg(&test_var, 0, 0), true); 664 KCSAN_EXPECT_RW_BARRIER(cmpxchg_release(&test_var, 0, 0), true); 665 KCSAN_EXPECT_RW_BARRIER(cmpxchg_relaxed(&test_var, 0, 0), false); 666 KCSAN_EXPECT_RW_BARRIER(atomic_read(&dummy), false); 667 KCSAN_EXPECT_RW_BARRIER(atomic_read_acquire(&dummy), false); 668 KCSAN_EXPECT_RW_BARRIER(atomic_set(&dummy, 0), false); 669 KCSAN_EXPECT_RW_BARRIER(atomic_set_release(&dummy, 0), true); 670 KCSAN_EXPECT_RW_BARRIER(atomic_add(1, &dummy), false); 671 KCSAN_EXPECT_RW_BARRIER(atomic_add_return(1, &dummy), true); 672 KCSAN_EXPECT_RW_BARRIER(atomic_add_return_acquire(1, &dummy), false); 673 KCSAN_EXPECT_RW_BARRIER(atomic_add_return_release(1, &dummy), true); 674 KCSAN_EXPECT_RW_BARRIER(atomic_add_return_relaxed(1, &dummy), false); 675 KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add(1, &dummy), true); 676 KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_acquire(1, &dummy), false); 677 KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_release(1, &dummy), true); 678 KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false); 679 KCSAN_EXPECT_RW_BARRIER(test_and_set_bit(0, &test_var), true); 680 KCSAN_EXPECT_RW_BARRIER(test_and_clear_bit(0, &test_var), true); 681 KCSAN_EXPECT_RW_BARRIER(test_and_change_bit(0, &test_var), true); 682 KCSAN_EXPECT_RW_BARRIER(clear_bit_unlock(0, &test_var), true); 683 KCSAN_EXPECT_RW_BARRIER(__clear_bit_unlock(0, &test_var), true); 684 KCSAN_EXPECT_RW_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var), true); 685 KCSAN_EXPECT_RW_BARRIER(arch_spin_lock(&arch_spinlock), false); 686 KCSAN_EXPECT_RW_BARRIER(arch_spin_unlock(&arch_spinlock), true); 687 KCSAN_EXPECT_RW_BARRIER(spin_lock(&spinlock), false); 688 KCSAN_EXPECT_RW_BARRIER(spin_unlock(&spinlock), true); 689 KCSAN_EXPECT_RW_BARRIER(mutex_lock(&mutex), false); 690 KCSAN_EXPECT_RW_BARRIER(mutex_unlock(&mutex), true); 691 692 kcsan_nestable_atomic_end(); 693 } 694 695 /* Simple test with normal data race. */ 696 __no_kcsan 697 static void test_basic(struct kunit *test) 698 { 699 struct expect_report expect = { 700 .access = { 701 { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 702 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 703 }, 704 }; 705 struct expect_report never = { 706 .access = { 707 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 708 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 709 }, 710 }; 711 bool match_expect = false; 712 bool match_never = false; 713 714 begin_test_checks(test_kernel_write, test_kernel_read); 715 do { 716 match_expect |= report_matches(&expect); 717 match_never = report_matches(&never); 718 } while (!end_test_checks(match_never)); 719 KUNIT_EXPECT_TRUE(test, match_expect); 720 KUNIT_EXPECT_FALSE(test, match_never); 721 } 722 723 /* 724 * Stress KCSAN with lots of concurrent races on different addresses until 725 * timeout. 726 */ 727 __no_kcsan 728 static void test_concurrent_races(struct kunit *test) 729 { 730 struct expect_report expect = { 731 .access = { 732 /* NULL will match any address. */ 733 { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, 734 { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(0) }, 735 }, 736 }; 737 struct expect_report never = { 738 .access = { 739 { test_kernel_rmw_array, NULL, 0, 0 }, 740 { test_kernel_rmw_array, NULL, 0, 0 }, 741 }, 742 }; 743 bool match_expect = false; 744 bool match_never = false; 745 746 begin_test_checks(test_kernel_rmw_array, test_kernel_rmw_array); 747 do { 748 match_expect |= report_matches(&expect); 749 match_never |= report_matches(&never); 750 } while (!end_test_checks(false)); 751 KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check matches exist. */ 752 KUNIT_EXPECT_FALSE(test, match_never); 753 } 754 755 /* Test the KCSAN_REPORT_VALUE_CHANGE_ONLY option. */ 756 __no_kcsan 757 static void test_novalue_change(struct kunit *test) 758 { 759 struct expect_report expect_rw = { 760 .access = { 761 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 762 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 763 }, 764 }; 765 struct expect_report expect_ww = { 766 .access = { 767 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 768 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 769 }, 770 }; 771 bool match_expect = false; 772 773 test_kernel_write_nochange(); /* Reset value. */ 774 begin_test_checks(test_kernel_write_nochange, test_kernel_read); 775 do { 776 match_expect = report_matches(&expect_rw) || report_matches(&expect_ww); 777 } while (!end_test_checks(match_expect)); 778 if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY)) 779 KUNIT_EXPECT_FALSE(test, match_expect); 780 else 781 KUNIT_EXPECT_TRUE(test, match_expect); 782 } 783 784 /* 785 * Test that the rules where the KCSAN_REPORT_VALUE_CHANGE_ONLY option should 786 * never apply work. 787 */ 788 __no_kcsan 789 static void test_novalue_change_exception(struct kunit *test) 790 { 791 struct expect_report expect_rw = { 792 .access = { 793 { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 794 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 795 }, 796 }; 797 struct expect_report expect_ww = { 798 .access = { 799 { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 800 { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 801 }, 802 }; 803 bool match_expect = false; 804 805 test_kernel_write_nochange_rcu(); /* Reset value. */ 806 begin_test_checks(test_kernel_write_nochange_rcu, test_kernel_read); 807 do { 808 match_expect = report_matches(&expect_rw) || report_matches(&expect_ww); 809 } while (!end_test_checks(match_expect)); 810 KUNIT_EXPECT_TRUE(test, match_expect); 811 } 812 813 /* Test that data races of unknown origin are reported. */ 814 __no_kcsan 815 static void test_unknown_origin(struct kunit *test) 816 { 817 struct expect_report expect = { 818 .access = { 819 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 820 { NULL }, 821 }, 822 }; 823 bool match_expect = false; 824 825 begin_test_checks(test_kernel_write_uninstrumented, test_kernel_read); 826 do { 827 match_expect = report_matches(&expect); 828 } while (!end_test_checks(match_expect)); 829 if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN)) 830 KUNIT_EXPECT_TRUE(test, match_expect); 831 else 832 KUNIT_EXPECT_FALSE(test, match_expect); 833 } 834 835 /* Test KCSAN_ASSUME_PLAIN_WRITES_ATOMIC if it is selected. */ 836 __no_kcsan 837 static void test_write_write_assume_atomic(struct kunit *test) 838 { 839 struct expect_report expect = { 840 .access = { 841 { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 842 { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 843 }, 844 }; 845 bool match_expect = false; 846 847 begin_test_checks(test_kernel_write, test_kernel_write); 848 do { 849 sink_value(READ_ONCE(test_var)); /* induce value-change */ 850 match_expect = report_matches(&expect); 851 } while (!end_test_checks(match_expect)); 852 if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) 853 KUNIT_EXPECT_FALSE(test, match_expect); 854 else 855 KUNIT_EXPECT_TRUE(test, match_expect); 856 } 857 858 /* 859 * Test that data races with writes larger than word-size are always reported, 860 * even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected. 861 */ 862 __no_kcsan 863 static void test_write_write_struct(struct kunit *test) 864 { 865 struct expect_report expect = { 866 .access = { 867 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, 868 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, 869 }, 870 }; 871 bool match_expect = false; 872 873 begin_test_checks(test_kernel_write_struct, test_kernel_write_struct); 874 do { 875 match_expect = report_matches(&expect); 876 } while (!end_test_checks(match_expect)); 877 KUNIT_EXPECT_TRUE(test, match_expect); 878 } 879 880 /* 881 * Test that data races where only one write is larger than word-size are always 882 * reported, even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected. 883 */ 884 __no_kcsan 885 static void test_write_write_struct_part(struct kunit *test) 886 { 887 struct expect_report expect = { 888 .access = { 889 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, 890 { test_kernel_write_struct_part, &test_struct.val[3], sizeof(test_struct.val[3]), KCSAN_ACCESS_WRITE }, 891 }, 892 }; 893 bool match_expect = false; 894 895 begin_test_checks(test_kernel_write_struct, test_kernel_write_struct_part); 896 do { 897 match_expect = report_matches(&expect); 898 } while (!end_test_checks(match_expect)); 899 KUNIT_EXPECT_TRUE(test, match_expect); 900 } 901 902 /* Test that races with atomic accesses never result in reports. */ 903 __no_kcsan 904 static void test_read_atomic_write_atomic(struct kunit *test) 905 { 906 bool match_never = false; 907 908 begin_test_checks(test_kernel_read_atomic, test_kernel_write_atomic); 909 do { 910 match_never = report_available(); 911 } while (!end_test_checks(match_never)); 912 KUNIT_EXPECT_FALSE(test, match_never); 913 } 914 915 /* Test that a race with an atomic and plain access result in reports. */ 916 __no_kcsan 917 static void test_read_plain_atomic_write(struct kunit *test) 918 { 919 struct expect_report expect = { 920 .access = { 921 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 922 { test_kernel_write_atomic, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC }, 923 }, 924 }; 925 bool match_expect = false; 926 927 KCSAN_TEST_REQUIRES(test, !IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)); 928 929 begin_test_checks(test_kernel_read, test_kernel_write_atomic); 930 do { 931 match_expect = report_matches(&expect); 932 } while (!end_test_checks(match_expect)); 933 KUNIT_EXPECT_TRUE(test, match_expect); 934 } 935 936 /* Test that atomic RMWs generate correct report. */ 937 __no_kcsan 938 static void test_read_plain_atomic_rmw(struct kunit *test) 939 { 940 struct expect_report expect = { 941 .access = { 942 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 943 { test_kernel_atomic_rmw, &test_var, sizeof(test_var), 944 KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC }, 945 }, 946 }; 947 bool match_expect = false; 948 949 KCSAN_TEST_REQUIRES(test, !IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)); 950 951 begin_test_checks(test_kernel_read, test_kernel_atomic_rmw); 952 do { 953 match_expect = report_matches(&expect); 954 } while (!end_test_checks(match_expect)); 955 KUNIT_EXPECT_TRUE(test, match_expect); 956 } 957 958 /* Zero-sized accesses should never cause data race reports. */ 959 __no_kcsan 960 static void test_zero_size_access(struct kunit *test) 961 { 962 struct expect_report expect = { 963 .access = { 964 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, 965 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, 966 }, 967 }; 968 struct expect_report never = { 969 .access = { 970 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, 971 { test_kernel_read_struct_zero_size, &test_struct.val[3], 0, 0 }, 972 }, 973 }; 974 bool match_expect = false; 975 bool match_never = false; 976 977 begin_test_checks(test_kernel_write_struct, test_kernel_read_struct_zero_size); 978 do { 979 match_expect |= report_matches(&expect); 980 match_never = report_matches(&never); 981 } while (!end_test_checks(match_never)); 982 KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check. */ 983 KUNIT_EXPECT_FALSE(test, match_never); 984 } 985 986 /* Test the data_race() macro. */ 987 __no_kcsan 988 static void test_data_race(struct kunit *test) 989 { 990 bool match_never = false; 991 992 begin_test_checks(test_kernel_data_race, test_kernel_data_race); 993 do { 994 match_never = report_available(); 995 } while (!end_test_checks(match_never)); 996 KUNIT_EXPECT_FALSE(test, match_never); 997 } 998 999 __no_kcsan 1000 static void test_assert_exclusive_writer(struct kunit *test) 1001 { 1002 struct expect_report expect = { 1003 .access = { 1004 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, 1005 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 1006 }, 1007 }; 1008 bool match_expect = false; 1009 1010 begin_test_checks(test_kernel_assert_writer, test_kernel_write_nochange); 1011 do { 1012 match_expect = report_matches(&expect); 1013 } while (!end_test_checks(match_expect)); 1014 KUNIT_EXPECT_TRUE(test, match_expect); 1015 } 1016 1017 __no_kcsan 1018 static void test_assert_exclusive_access(struct kunit *test) 1019 { 1020 struct expect_report expect = { 1021 .access = { 1022 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, 1023 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 1024 }, 1025 }; 1026 bool match_expect = false; 1027 1028 begin_test_checks(test_kernel_assert_access, test_kernel_read); 1029 do { 1030 match_expect = report_matches(&expect); 1031 } while (!end_test_checks(match_expect)); 1032 KUNIT_EXPECT_TRUE(test, match_expect); 1033 } 1034 1035 __no_kcsan 1036 static void test_assert_exclusive_access_writer(struct kunit *test) 1037 { 1038 struct expect_report expect_access_writer = { 1039 .access = { 1040 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, 1041 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, 1042 }, 1043 }; 1044 struct expect_report expect_access_access = { 1045 .access = { 1046 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, 1047 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, 1048 }, 1049 }; 1050 struct expect_report never = { 1051 .access = { 1052 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, 1053 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, 1054 }, 1055 }; 1056 bool match_expect_access_writer = false; 1057 bool match_expect_access_access = false; 1058 bool match_never = false; 1059 1060 begin_test_checks(test_kernel_assert_access, test_kernel_assert_writer); 1061 do { 1062 match_expect_access_writer |= report_matches(&expect_access_writer); 1063 match_expect_access_access |= report_matches(&expect_access_access); 1064 match_never |= report_matches(&never); 1065 } while (!end_test_checks(match_never)); 1066 KUNIT_EXPECT_TRUE(test, match_expect_access_writer); 1067 KUNIT_EXPECT_TRUE(test, match_expect_access_access); 1068 KUNIT_EXPECT_FALSE(test, match_never); 1069 } 1070 1071 __no_kcsan 1072 static void test_assert_exclusive_bits_change(struct kunit *test) 1073 { 1074 struct expect_report expect = { 1075 .access = { 1076 { test_kernel_assert_bits_change, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, 1077 { test_kernel_change_bits, &test_var, sizeof(test_var), 1078 KCSAN_ACCESS_WRITE | (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) ? 0 : KCSAN_ACCESS_ATOMIC) }, 1079 }, 1080 }; 1081 bool match_expect = false; 1082 1083 begin_test_checks(test_kernel_assert_bits_change, test_kernel_change_bits); 1084 do { 1085 match_expect = report_matches(&expect); 1086 } while (!end_test_checks(match_expect)); 1087 KUNIT_EXPECT_TRUE(test, match_expect); 1088 } 1089 1090 __no_kcsan 1091 static void test_assert_exclusive_bits_nochange(struct kunit *test) 1092 { 1093 bool match_never = false; 1094 1095 begin_test_checks(test_kernel_assert_bits_nochange, test_kernel_change_bits); 1096 do { 1097 match_never = report_available(); 1098 } while (!end_test_checks(match_never)); 1099 KUNIT_EXPECT_FALSE(test, match_never); 1100 } 1101 1102 __no_kcsan 1103 static void test_assert_exclusive_writer_scoped(struct kunit *test) 1104 { 1105 struct expect_report expect_start = { 1106 .access = { 1107 { test_kernel_assert_writer_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED }, 1108 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 1109 }, 1110 }; 1111 struct expect_report expect_inscope = { 1112 .access = { 1113 { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED }, 1114 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 1115 }, 1116 }; 1117 bool match_expect_start = false; 1118 bool match_expect_inscope = false; 1119 1120 begin_test_checks(test_kernel_assert_writer_scoped, test_kernel_write_nochange); 1121 do { 1122 match_expect_start |= report_matches(&expect_start); 1123 match_expect_inscope |= report_matches(&expect_inscope); 1124 } while (!end_test_checks(match_expect_inscope)); 1125 KUNIT_EXPECT_TRUE(test, match_expect_start); 1126 KUNIT_EXPECT_FALSE(test, match_expect_inscope); 1127 } 1128 1129 __no_kcsan 1130 static void test_assert_exclusive_access_scoped(struct kunit *test) 1131 { 1132 struct expect_report expect_start1 = { 1133 .access = { 1134 { test_kernel_assert_access_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED }, 1135 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 1136 }, 1137 }; 1138 struct expect_report expect_start2 = { 1139 .access = { expect_start1.access[0], expect_start1.access[0] }, 1140 }; 1141 struct expect_report expect_inscope = { 1142 .access = { 1143 { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED }, 1144 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 1145 }, 1146 }; 1147 bool match_expect_start = false; 1148 bool match_expect_inscope = false; 1149 1150 begin_test_checks(test_kernel_assert_access_scoped, test_kernel_read); 1151 end_time += msecs_to_jiffies(1000); /* This test requires a bit more time. */ 1152 do { 1153 match_expect_start |= report_matches(&expect_start1) || report_matches(&expect_start2); 1154 match_expect_inscope |= report_matches(&expect_inscope); 1155 } while (!end_test_checks(match_expect_inscope)); 1156 KUNIT_EXPECT_TRUE(test, match_expect_start); 1157 KUNIT_EXPECT_FALSE(test, match_expect_inscope); 1158 } 1159 1160 /* 1161 * jiffies is special (declared to be volatile) and its accesses are typically 1162 * not marked; this test ensures that the compiler nor KCSAN gets confused about 1163 * jiffies's declaration on different architectures. 1164 */ 1165 __no_kcsan 1166 static void test_jiffies_noreport(struct kunit *test) 1167 { 1168 bool match_never = false; 1169 1170 begin_test_checks(test_kernel_jiffies_reader, test_kernel_jiffies_reader); 1171 do { 1172 match_never = report_available(); 1173 } while (!end_test_checks(match_never)); 1174 KUNIT_EXPECT_FALSE(test, match_never); 1175 } 1176 1177 /* Test that racing accesses in seqlock critical sections are not reported. */ 1178 __no_kcsan 1179 static void test_seqlock_noreport(struct kunit *test) 1180 { 1181 bool match_never = false; 1182 1183 begin_test_checks(test_kernel_seqlock_reader, test_kernel_seqlock_writer); 1184 do { 1185 match_never = report_available(); 1186 } while (!end_test_checks(match_never)); 1187 KUNIT_EXPECT_FALSE(test, match_never); 1188 } 1189 1190 /* 1191 * Test atomic builtins work and required instrumentation functions exist. We 1192 * also test that KCSAN understands they're atomic by racing with them via 1193 * test_kernel_atomic_builtins(), and expect no reports. 1194 * 1195 * The atomic builtins _SHOULD NOT_ be used in normal kernel code! 1196 */ 1197 static void test_atomic_builtins(struct kunit *test) 1198 { 1199 bool match_never = false; 1200 1201 begin_test_checks(test_kernel_atomic_builtins, test_kernel_atomic_builtins); 1202 do { 1203 long tmp; 1204 1205 kcsan_enable_current(); 1206 1207 __atomic_store_n(&test_var, 42L, __ATOMIC_RELAXED); 1208 KUNIT_EXPECT_EQ(test, 42L, __atomic_load_n(&test_var, __ATOMIC_RELAXED)); 1209 1210 KUNIT_EXPECT_EQ(test, 42L, __atomic_exchange_n(&test_var, 20, __ATOMIC_RELAXED)); 1211 KUNIT_EXPECT_EQ(test, 20L, test_var); 1212 1213 tmp = 20L; 1214 KUNIT_EXPECT_TRUE(test, __atomic_compare_exchange_n(&test_var, &tmp, 30L, 1215 0, __ATOMIC_RELAXED, 1216 __ATOMIC_RELAXED)); 1217 KUNIT_EXPECT_EQ(test, tmp, 20L); 1218 KUNIT_EXPECT_EQ(test, test_var, 30L); 1219 KUNIT_EXPECT_FALSE(test, __atomic_compare_exchange_n(&test_var, &tmp, 40L, 1220 1, __ATOMIC_RELAXED, 1221 __ATOMIC_RELAXED)); 1222 KUNIT_EXPECT_EQ(test, tmp, 30L); 1223 KUNIT_EXPECT_EQ(test, test_var, 30L); 1224 1225 KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED)); 1226 KUNIT_EXPECT_EQ(test, 31L, __atomic_fetch_sub(&test_var, 1, __ATOMIC_RELAXED)); 1227 KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_and(&test_var, 0xf, __ATOMIC_RELAXED)); 1228 KUNIT_EXPECT_EQ(test, 14L, __atomic_fetch_xor(&test_var, 0xf, __ATOMIC_RELAXED)); 1229 KUNIT_EXPECT_EQ(test, 1L, __atomic_fetch_or(&test_var, 0xf0, __ATOMIC_RELAXED)); 1230 KUNIT_EXPECT_EQ(test, 241L, __atomic_fetch_nand(&test_var, 0xf, __ATOMIC_RELAXED)); 1231 KUNIT_EXPECT_EQ(test, -2L, test_var); 1232 1233 __atomic_thread_fence(__ATOMIC_SEQ_CST); 1234 __atomic_signal_fence(__ATOMIC_SEQ_CST); 1235 1236 kcsan_disable_current(); 1237 1238 match_never = report_available(); 1239 } while (!end_test_checks(match_never)); 1240 KUNIT_EXPECT_FALSE(test, match_never); 1241 } 1242 1243 __no_kcsan 1244 static void test_1bit_value_change(struct kunit *test) 1245 { 1246 struct expect_report expect = { 1247 .access = { 1248 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 1249 { test_kernel_xor_1bit, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, 1250 }, 1251 }; 1252 bool match = false; 1253 1254 begin_test_checks(test_kernel_read, test_kernel_xor_1bit); 1255 do { 1256 match = IS_ENABLED(CONFIG_KCSAN_PERMISSIVE) 1257 ? report_available() 1258 : report_matches(&expect); 1259 } while (!end_test_checks(match)); 1260 if (IS_ENABLED(CONFIG_KCSAN_PERMISSIVE)) 1261 KUNIT_EXPECT_FALSE(test, match); 1262 else 1263 KUNIT_EXPECT_TRUE(test, match); 1264 } 1265 1266 __no_kcsan 1267 static void test_correct_barrier(struct kunit *test) 1268 { 1269 struct expect_report expect = { 1270 .access = { 1271 { test_kernel_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, 1272 { test_kernel_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, 1273 }, 1274 }; 1275 bool match_expect = false; 1276 1277 test_struct.val[0] = 0; /* init unlocked */ 1278 begin_test_checks(test_kernel_with_memorder, test_kernel_with_memorder); 1279 do { 1280 match_expect = report_matches_any_reordered(&expect); 1281 } while (!end_test_checks(match_expect)); 1282 KUNIT_EXPECT_FALSE(test, match_expect); 1283 } 1284 1285 __no_kcsan 1286 static void test_missing_barrier(struct kunit *test) 1287 { 1288 struct expect_report expect = { 1289 .access = { 1290 { test_kernel_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, 1291 { test_kernel_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, 1292 }, 1293 }; 1294 bool match_expect = false; 1295 1296 test_struct.val[0] = 0; /* init unlocked */ 1297 begin_test_checks(test_kernel_wrong_memorder, test_kernel_wrong_memorder); 1298 do { 1299 match_expect = report_matches_any_reordered(&expect); 1300 } while (!end_test_checks(match_expect)); 1301 if (IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY)) 1302 KUNIT_EXPECT_TRUE(test, match_expect); 1303 else 1304 KUNIT_EXPECT_FALSE(test, match_expect); 1305 } 1306 1307 __no_kcsan 1308 static void test_atomic_builtins_correct_barrier(struct kunit *test) 1309 { 1310 struct expect_report expect = { 1311 .access = { 1312 { test_kernel_atomic_builtin_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, 1313 { test_kernel_atomic_builtin_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, 1314 }, 1315 }; 1316 bool match_expect = false; 1317 1318 test_struct.val[0] = 0; /* init unlocked */ 1319 begin_test_checks(test_kernel_atomic_builtin_with_memorder, 1320 test_kernel_atomic_builtin_with_memorder); 1321 do { 1322 match_expect = report_matches_any_reordered(&expect); 1323 } while (!end_test_checks(match_expect)); 1324 KUNIT_EXPECT_FALSE(test, match_expect); 1325 } 1326 1327 __no_kcsan 1328 static void test_atomic_builtins_missing_barrier(struct kunit *test) 1329 { 1330 struct expect_report expect = { 1331 .access = { 1332 { test_kernel_atomic_builtin_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, 1333 { test_kernel_atomic_builtin_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, 1334 }, 1335 }; 1336 bool match_expect = false; 1337 1338 test_struct.val[0] = 0; /* init unlocked */ 1339 begin_test_checks(test_kernel_atomic_builtin_wrong_memorder, 1340 test_kernel_atomic_builtin_wrong_memorder); 1341 do { 1342 match_expect = report_matches_any_reordered(&expect); 1343 } while (!end_test_checks(match_expect)); 1344 if (IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY)) 1345 KUNIT_EXPECT_TRUE(test, match_expect); 1346 else 1347 KUNIT_EXPECT_FALSE(test, match_expect); 1348 } 1349 1350 /* 1351 * Generate thread counts for all test cases. Values generated are in interval 1352 * [2, 5] followed by exponentially increasing thread counts from 8 to 32. 1353 * 1354 * The thread counts are chosen to cover potentially interesting boundaries and 1355 * corner cases (2 to 5), and then stress the system with larger counts. 1356 */ 1357 static const void *nthreads_gen_params(const void *prev, char *desc) 1358 { 1359 long nthreads = (long)prev; 1360 1361 if (nthreads < 0 || nthreads >= 32) 1362 nthreads = 0; /* stop */ 1363 else if (!nthreads) 1364 nthreads = 2; /* initial value */ 1365 else if (nthreads < 5) 1366 nthreads++; 1367 else if (nthreads == 5) 1368 nthreads = 8; 1369 else 1370 nthreads *= 2; 1371 1372 if (!IS_ENABLED(CONFIG_PREEMPT) || !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) { 1373 /* 1374 * Without any preemption, keep 2 CPUs free for other tasks, one 1375 * of which is the main test case function checking for 1376 * completion or failure. 1377 */ 1378 const long min_unused_cpus = IS_ENABLED(CONFIG_PREEMPT_NONE) ? 2 : 0; 1379 const long min_required_cpus = 2 + min_unused_cpus; 1380 1381 if (num_online_cpus() < min_required_cpus) { 1382 pr_err_once("Too few online CPUs (%u < %ld) for test\n", 1383 num_online_cpus(), min_required_cpus); 1384 nthreads = 0; 1385 } else if (nthreads >= num_online_cpus() - min_unused_cpus) { 1386 /* Use negative value to indicate last param. */ 1387 nthreads = -(num_online_cpus() - min_unused_cpus); 1388 pr_warn_once("Limiting number of threads to %ld (only %d online CPUs)\n", 1389 -nthreads, num_online_cpus()); 1390 } 1391 } 1392 1393 snprintf(desc, KUNIT_PARAM_DESC_SIZE, "threads=%ld", abs(nthreads)); 1394 return (void *)nthreads; 1395 } 1396 1397 #define KCSAN_KUNIT_CASE(test_name) KUNIT_CASE_PARAM(test_name, nthreads_gen_params) 1398 static struct kunit_case kcsan_test_cases[] = { 1399 KUNIT_CASE(test_barrier_nothreads), 1400 KCSAN_KUNIT_CASE(test_basic), 1401 KCSAN_KUNIT_CASE(test_concurrent_races), 1402 KCSAN_KUNIT_CASE(test_novalue_change), 1403 KCSAN_KUNIT_CASE(test_novalue_change_exception), 1404 KCSAN_KUNIT_CASE(test_unknown_origin), 1405 KCSAN_KUNIT_CASE(test_write_write_assume_atomic), 1406 KCSAN_KUNIT_CASE(test_write_write_struct), 1407 KCSAN_KUNIT_CASE(test_write_write_struct_part), 1408 KCSAN_KUNIT_CASE(test_read_atomic_write_atomic), 1409 KCSAN_KUNIT_CASE(test_read_plain_atomic_write), 1410 KCSAN_KUNIT_CASE(test_read_plain_atomic_rmw), 1411 KCSAN_KUNIT_CASE(test_zero_size_access), 1412 KCSAN_KUNIT_CASE(test_data_race), 1413 KCSAN_KUNIT_CASE(test_assert_exclusive_writer), 1414 KCSAN_KUNIT_CASE(test_assert_exclusive_access), 1415 KCSAN_KUNIT_CASE(test_assert_exclusive_access_writer), 1416 KCSAN_KUNIT_CASE(test_assert_exclusive_bits_change), 1417 KCSAN_KUNIT_CASE(test_assert_exclusive_bits_nochange), 1418 KCSAN_KUNIT_CASE(test_assert_exclusive_writer_scoped), 1419 KCSAN_KUNIT_CASE(test_assert_exclusive_access_scoped), 1420 KCSAN_KUNIT_CASE(test_jiffies_noreport), 1421 KCSAN_KUNIT_CASE(test_seqlock_noreport), 1422 KCSAN_KUNIT_CASE(test_atomic_builtins), 1423 KCSAN_KUNIT_CASE(test_1bit_value_change), 1424 KCSAN_KUNIT_CASE(test_correct_barrier), 1425 KCSAN_KUNIT_CASE(test_missing_barrier), 1426 KCSAN_KUNIT_CASE(test_atomic_builtins_correct_barrier), 1427 KCSAN_KUNIT_CASE(test_atomic_builtins_missing_barrier), 1428 {}, 1429 }; 1430 1431 /* ===== End test cases ===== */ 1432 1433 /* Concurrent accesses from interrupts. */ 1434 __no_kcsan 1435 static void access_thread_timer(struct timer_list *timer) 1436 { 1437 static atomic_t cnt = ATOMIC_INIT(0); 1438 unsigned int idx; 1439 void (*func)(void); 1440 1441 idx = (unsigned int)atomic_inc_return(&cnt) % ARRAY_SIZE(access_kernels); 1442 /* Acquire potential initialization. */ 1443 func = smp_load_acquire(&access_kernels[idx]); 1444 if (func) 1445 func(); 1446 } 1447 1448 /* The main loop for each thread. */ 1449 __no_kcsan 1450 static int access_thread(void *arg) 1451 { 1452 struct timer_list timer; 1453 unsigned int cnt = 0; 1454 unsigned int idx; 1455 void (*func)(void); 1456 1457 timer_setup_on_stack(&timer, access_thread_timer, 0); 1458 do { 1459 might_sleep(); 1460 1461 if (!timer_pending(&timer)) 1462 mod_timer(&timer, jiffies + 1); 1463 else { 1464 /* Iterate through all kernels. */ 1465 idx = cnt++ % ARRAY_SIZE(access_kernels); 1466 /* Acquire potential initialization. */ 1467 func = smp_load_acquire(&access_kernels[idx]); 1468 if (func) 1469 func(); 1470 } 1471 } while (!torture_must_stop()); 1472 del_timer_sync(&timer); 1473 destroy_timer_on_stack(&timer); 1474 1475 torture_kthread_stopping("access_thread"); 1476 return 0; 1477 } 1478 1479 __no_kcsan 1480 static int test_init(struct kunit *test) 1481 { 1482 unsigned long flags; 1483 int nthreads; 1484 int i; 1485 1486 spin_lock_irqsave(&observed.lock, flags); 1487 for (i = 0; i < ARRAY_SIZE(observed.lines); ++i) 1488 observed.lines[i][0] = '\0'; 1489 observed.nlines = 0; 1490 spin_unlock_irqrestore(&observed.lock, flags); 1491 1492 if (strstr(test->name, "nothreads")) 1493 return 0; 1494 1495 if (!torture_init_begin((char *)test->name, 1)) 1496 return -EBUSY; 1497 1498 if (WARN_ON(threads)) 1499 goto err; 1500 1501 for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) { 1502 if (WARN_ON(access_kernels[i])) 1503 goto err; 1504 } 1505 1506 nthreads = abs((long)test->param_value); 1507 if (WARN_ON(!nthreads)) 1508 goto err; 1509 1510 threads = kcalloc(nthreads + 1, sizeof(struct task_struct *), GFP_KERNEL); 1511 if (WARN_ON(!threads)) 1512 goto err; 1513 1514 threads[nthreads] = NULL; 1515 for (i = 0; i < nthreads; ++i) { 1516 if (torture_create_kthread(access_thread, NULL, threads[i])) 1517 goto err; 1518 } 1519 1520 torture_init_end(); 1521 1522 return 0; 1523 1524 err: 1525 kfree(threads); 1526 threads = NULL; 1527 torture_init_end(); 1528 return -EINVAL; 1529 } 1530 1531 __no_kcsan 1532 static void test_exit(struct kunit *test) 1533 { 1534 struct task_struct **stop_thread; 1535 int i; 1536 1537 if (strstr(test->name, "nothreads")) 1538 return; 1539 1540 if (torture_cleanup_begin()) 1541 return; 1542 1543 for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) 1544 WRITE_ONCE(access_kernels[i], NULL); 1545 1546 if (threads) { 1547 for (stop_thread = threads; *stop_thread; stop_thread++) 1548 torture_stop_kthread(reader_thread, *stop_thread); 1549 1550 kfree(threads); 1551 threads = NULL; 1552 } 1553 1554 torture_cleanup_end(); 1555 } 1556 1557 static struct kunit_suite kcsan_test_suite = { 1558 .name = "kcsan", 1559 .test_cases = kcsan_test_cases, 1560 .init = test_init, 1561 .exit = test_exit, 1562 }; 1563 static struct kunit_suite *kcsan_test_suites[] = { &kcsan_test_suite, NULL }; 1564 1565 __no_kcsan 1566 static void register_tracepoints(struct tracepoint *tp, void *ignore) 1567 { 1568 check_trace_callback_type_console(probe_console); 1569 if (!strcmp(tp->name, "console")) 1570 WARN_ON(tracepoint_probe_register(tp, probe_console, NULL)); 1571 } 1572 1573 __no_kcsan 1574 static void unregister_tracepoints(struct tracepoint *tp, void *ignore) 1575 { 1576 if (!strcmp(tp->name, "console")) 1577 tracepoint_probe_unregister(tp, probe_console, NULL); 1578 } 1579 1580 /* 1581 * We only want to do tracepoints setup and teardown once, therefore we have to 1582 * customize the init and exit functions and cannot rely on kunit_test_suite(). 1583 */ 1584 static int __init kcsan_test_init(void) 1585 { 1586 /* 1587 * Because we want to be able to build the test as a module, we need to 1588 * iterate through all known tracepoints, since the static registration 1589 * won't work here. 1590 */ 1591 for_each_kernel_tracepoint(register_tracepoints, NULL); 1592 return __kunit_test_suites_init(kcsan_test_suites); 1593 } 1594 1595 static void kcsan_test_exit(void) 1596 { 1597 __kunit_test_suites_exit(kcsan_test_suites); 1598 for_each_kernel_tracepoint(unregister_tracepoints, NULL); 1599 tracepoint_synchronize_unregister(); 1600 } 1601 1602 late_initcall_sync(kcsan_test_init); 1603 module_exit(kcsan_test_exit); 1604 1605 MODULE_LICENSE("GPL v2"); 1606 MODULE_AUTHOR("Marco Elver <[email protected]>"); 1607