1 //===-- tsan_interceptors_libdispatch.cpp ---------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // Support for intercepting libdispatch (GCD). 12 //===----------------------------------------------------------------------===// 13 14 #include "sanitizer_common/sanitizer_common.h" 15 #include "interception/interception.h" 16 #include "tsan_interceptors.h" 17 #include "tsan_rtl.h" 18 19 #include "BlocksRuntime/Block.h" 20 #include "tsan_dispatch_defs.h" 21 22 namespace __tsan { 23 typedef u16 uint16_t; 24 25 typedef struct { 26 dispatch_queue_t queue; 27 void *orig_context; 28 dispatch_function_t orig_work; 29 bool free_context_in_callback; 30 bool submitted_synchronously; 31 bool is_barrier_block; 32 uptr non_queue_sync_object; 33 } block_context_t; 34 35 // The offsets of different fields of the dispatch_queue_t structure, exported 36 // by libdispatch.dylib. 37 extern "C" struct dispatch_queue_offsets_s { 38 const uint16_t dqo_version; 39 const uint16_t dqo_label; 40 const uint16_t dqo_label_size; 41 const uint16_t dqo_flags; 42 const uint16_t dqo_flags_size; 43 const uint16_t dqo_serialnum; 44 const uint16_t dqo_serialnum_size; 45 const uint16_t dqo_width; 46 const uint16_t dqo_width_size; 47 const uint16_t dqo_running; 48 const uint16_t dqo_running_size; 49 const uint16_t dqo_suspend_cnt; 50 const uint16_t dqo_suspend_cnt_size; 51 const uint16_t dqo_target_queue; 52 const uint16_t dqo_target_queue_size; 53 const uint16_t dqo_priority; 54 const uint16_t dqo_priority_size; 55 } dispatch_queue_offsets; 56 57 static bool IsQueueSerial(dispatch_queue_t q) { 58 CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2); 59 uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width); 60 CHECK_NE(width, 0); 61 return width == 1; 62 } 63 64 static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) { 65 CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8); 66 dispatch_queue_t tq = *( 67 dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue); 68 return tq; 69 } 70 71 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { 72 dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source); 73 CHECK_NE(tq, 0); 74 return tq; 75 } 76 77 static block_context_t *AllocContext(ThreadState *thr, uptr pc, 78 dispatch_queue_t queue, void *orig_context, 79 dispatch_function_t orig_work) { 80 block_context_t *new_context = 81 (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t)); 82 new_context->queue = queue; 83 new_context->orig_context = orig_context; 84 new_context->orig_work = orig_work; 85 new_context->free_context_in_callback = true; 86 new_context->submitted_synchronously = false; 87 new_context->is_barrier_block = false; 88 new_context->non_queue_sync_object = 0; 89 return new_context; 90 } 91 92 #define GET_QUEUE_SYNC_VARS(context, q) \ 93 bool is_queue_serial = q && IsQueueSerial(q); \ 94 uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \ 95 uptr serial_sync = (uptr)sync_ptr; \ 96 uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \ 97 bool serial_task = context->is_barrier_block || is_queue_serial 98 99 static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, 100 block_context_t *context) { 101 uptr submit_sync = (uptr)context; 102 Acquire(thr, pc, submit_sync); 103 104 dispatch_queue_t q = context->queue; 105 do { 106 GET_QUEUE_SYNC_VARS(context, q); 107 if (serial_sync) Acquire(thr, pc, serial_sync); 108 if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync); 109 110 if (q) q = GetTargetQueueFromQueue(q); 111 } while (q); 112 } 113 114 static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, 115 block_context_t *context) { 116 uptr submit_sync = (uptr)context; 117 if (context->submitted_synchronously) Release(thr, pc, submit_sync); 118 119 dispatch_queue_t q = context->queue; 120 do { 121 GET_QUEUE_SYNC_VARS(context, q); 122 if (serial_task && serial_sync) Release(thr, pc, serial_sync); 123 if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync); 124 125 if (q) q = GetTargetQueueFromQueue(q); 126 } while (q); 127 } 128 129 static void dispatch_callback_wrap(void *param) { 130 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); 131 block_context_t *context = (block_context_t *)param; 132 133 dispatch_sync_pre_execute(thr, pc, context); 134 135 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 136 context->orig_work(context->orig_context); 137 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 138 139 dispatch_sync_post_execute(thr, pc, context); 140 141 if (context->free_context_in_callback) user_free(thr, pc, context); 142 } 143 144 static void invoke_block(void *param) { 145 dispatch_block_t block = (dispatch_block_t)param; 146 block(); 147 } 148 149 static void invoke_and_release_block(void *param) { 150 dispatch_block_t block = (dispatch_block_t)param; 151 block(); 152 Block_release(block); 153 } 154 155 #define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \ 156 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ 157 SCOPED_TSAN_INTERCEPTOR(name, q, block); \ 158 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 159 dispatch_block_t heap_block = Block_copy(block); \ 160 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 161 block_context_t *new_context = \ 162 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ 163 new_context->is_barrier_block = barrier; \ 164 Release(thr, pc, (uptr)new_context); \ 165 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 166 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \ 167 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 168 } 169 170 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \ 171 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ 172 DISPATCH_NOESCAPE dispatch_block_t block) { \ 173 SCOPED_TSAN_INTERCEPTOR(name, q, block); \ 174 block_context_t new_context = { \ 175 q, block, &invoke_block, false, true, barrier, 0}; \ 176 Release(thr, pc, (uptr)&new_context); \ 177 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 178 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \ 179 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 180 Acquire(thr, pc, (uptr)&new_context); \ 181 } 182 183 #define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \ 184 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ 185 dispatch_function_t work) { \ 186 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ 187 block_context_t *new_context = \ 188 AllocContext(thr, pc, q, context, work); \ 189 new_context->is_barrier_block = barrier; \ 190 Release(thr, pc, (uptr)new_context); \ 191 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 192 REAL(name)(q, new_context, dispatch_callback_wrap); \ 193 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 194 } 195 196 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \ 197 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ 198 dispatch_function_t work) { \ 199 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ 200 block_context_t new_context = { \ 201 q, context, work, false, true, barrier, 0}; \ 202 Release(thr, pc, (uptr)&new_context); \ 203 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 204 REAL(name)(q, &new_context, dispatch_callback_wrap); \ 205 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 206 Acquire(thr, pc, (uptr)&new_context); \ 207 } 208 209 #define DISPATCH_INTERCEPT(name, barrier) \ 210 DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \ 211 DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \ 212 DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \ 213 DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier) 214 215 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new 216 // context, which is used to synchronize (we release the context before 217 // submitting, and the callback acquires it before executing the original 218 // callback). 219 DISPATCH_INTERCEPT(dispatch, false) 220 DISPATCH_INTERCEPT(dispatch_barrier, true) 221 222 DISPATCH_INTERCEPT_SYNC_F(dispatch_async_and_wait_f, false) 223 DISPATCH_INTERCEPT_SYNC_B(dispatch_async_and_wait, false) 224 225 DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when, 226 dispatch_queue_t queue, void *context, dispatch_function_t work) 227 228 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, 229 dispatch_queue_t queue, dispatch_block_t block) { 230 SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block); 231 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 232 dispatch_block_t heap_block = Block_copy(block); 233 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 234 block_context_t *new_context = 235 AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); 236 Release(thr, pc, (uptr)new_context); 237 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 238 REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap); 239 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 240 } 241 242 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, 243 dispatch_queue_t queue, void *context, 244 dispatch_function_t work) { 245 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work); 246 WRAP(dispatch_after)(when, queue, ^(void) { 247 work(context); 248 }); 249 } 250 251 // GCD's dispatch_once implementation has a fast path that contains a racy read 252 // and it's inlined into user's code. Furthermore, this fast path doesn't 253 // establish a proper happens-before relations between the initialization and 254 // code following the call to dispatch_once. We could deal with this in 255 // instrumented code, but there's not much we can do about it in system 256 // libraries. Let's disable the fast path (by never storing the value ~0 to 257 // predicate), so the interceptor is always called, and let's add proper release 258 // and acquire semantics. Since TSan does not see its own atomic stores, the 259 // race on predicate won't be reported - the only accesses to it that TSan sees 260 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is 261 // both a macro and a real function, we want to intercept the function, so we 262 // need to undefine the macro. 263 #undef dispatch_once 264 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, 265 DISPATCH_NOESCAPE dispatch_block_t block) { 266 SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block); 267 atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate); 268 u32 v = atomic_load(a, memory_order_acquire); 269 if (v == 0 && 270 atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { 271 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 272 block(); 273 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 274 Release(thr, pc, (uptr)a); 275 atomic_store(a, 2, memory_order_release); 276 } else { 277 while (v != 2) { 278 internal_sched_yield(); 279 v = atomic_load(a, memory_order_acquire); 280 } 281 Acquire(thr, pc, (uptr)a); 282 } 283 } 284 285 #undef dispatch_once_f 286 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate, 287 void *context, dispatch_function_t function) { 288 SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function); 289 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 290 WRAP(dispatch_once)(predicate, ^(void) { 291 function(context); 292 }); 293 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 294 } 295 296 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal, 297 dispatch_semaphore_t dsema) { 298 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema); 299 Release(thr, pc, (uptr)dsema); 300 return REAL(dispatch_semaphore_signal)(dsema); 301 } 302 303 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema, 304 dispatch_time_t timeout) { 305 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout); 306 long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout); 307 if (result == 0) Acquire(thr, pc, (uptr)dsema); 308 return result; 309 } 310 311 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, 312 dispatch_time_t timeout) { 313 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout); 314 long_t result = REAL(dispatch_group_wait)(group, timeout); 315 if (result == 0) Acquire(thr, pc, (uptr)group); 316 return result; 317 } 318 319 // Used, but not intercepted. 320 extern "C" void dispatch_group_enter(dispatch_group_t group); 321 322 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { 323 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); 324 // Acquired in the group notification callback in dispatch_group_notify[_f]. 325 Release(thr, pc, (uptr)group); 326 REAL(dispatch_group_leave)(group); 327 } 328 329 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, 330 dispatch_queue_t queue, dispatch_block_t block) { 331 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); 332 dispatch_retain(group); 333 dispatch_group_enter(group); 334 __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block); 335 WRAP(dispatch_async)(queue, ^(void) { 336 block_copy(); 337 Block_release(block_copy); 338 WRAP(dispatch_group_leave)(group); 339 dispatch_release(group); 340 }); 341 } 342 343 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, 344 dispatch_queue_t queue, void *context, 345 dispatch_function_t work) { 346 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work); 347 dispatch_retain(group); 348 dispatch_group_enter(group); 349 WRAP(dispatch_async)(queue, ^(void) { 350 work(context); 351 WRAP(dispatch_group_leave)(group); 352 dispatch_release(group); 353 }); 354 } 355 356 DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group, 357 dispatch_queue_t q, void *context, dispatch_function_t work) 358 359 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, 360 dispatch_queue_t q, dispatch_block_t block) { 361 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); 362 363 // To make sure the group is still available in the callback (otherwise 364 // it can be already destroyed). Will be released in the callback. 365 dispatch_retain(group); 366 367 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 368 dispatch_block_t heap_block = Block_copy(^(void) { 369 { 370 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback); 371 // Released when leaving the group (dispatch_group_leave). 372 Acquire(thr, pc, (uptr)group); 373 } 374 dispatch_release(group); 375 block(); 376 }); 377 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 378 block_context_t *new_context = 379 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); 380 new_context->is_barrier_block = true; 381 Release(thr, pc, (uptr)new_context); 382 REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap); 383 } 384 385 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group, 386 dispatch_queue_t q, void *context, dispatch_function_t work) { 387 WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); }); 388 } 389 390 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, 391 dispatch_source_t source, dispatch_block_t handler) { 392 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler); 393 if (handler == nullptr) 394 return REAL(dispatch_source_set_event_handler)(source, nullptr); 395 dispatch_queue_t q = GetTargetQueueFromSource(source); 396 __block block_context_t new_context = { 397 q, handler, &invoke_block, false, false, false, 0 }; 398 dispatch_block_t new_handler = Block_copy(^(void) { 399 new_context.orig_context = handler; // To explicitly capture "handler". 400 dispatch_callback_wrap(&new_context); 401 }); 402 uptr submit_sync = (uptr)&new_context; 403 Release(thr, pc, submit_sync); 404 REAL(dispatch_source_set_event_handler)(source, new_handler); 405 Block_release(new_handler); 406 } 407 408 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f, 409 dispatch_source_t source, dispatch_function_t handler) { 410 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler); 411 if (handler == nullptr) 412 return REAL(dispatch_source_set_event_handler)(source, nullptr); 413 dispatch_block_t block = ^(void) { 414 handler(dispatch_get_context(source)); 415 }; 416 WRAP(dispatch_source_set_event_handler)(source, block); 417 } 418 419 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, 420 dispatch_source_t source, dispatch_block_t handler) { 421 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler); 422 if (handler == nullptr) 423 return REAL(dispatch_source_set_cancel_handler)(source, nullptr); 424 dispatch_queue_t q = GetTargetQueueFromSource(source); 425 __block block_context_t new_context = { 426 q, handler, &invoke_block, false, false, false, 0}; 427 dispatch_block_t new_handler = Block_copy(^(void) { 428 new_context.orig_context = handler; // To explicitly capture "handler". 429 dispatch_callback_wrap(&new_context); 430 }); 431 uptr submit_sync = (uptr)&new_context; 432 Release(thr, pc, submit_sync); 433 REAL(dispatch_source_set_cancel_handler)(source, new_handler); 434 Block_release(new_handler); 435 } 436 437 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f, 438 dispatch_source_t source, dispatch_function_t handler) { 439 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source, 440 handler); 441 if (handler == nullptr) 442 return REAL(dispatch_source_set_cancel_handler)(source, nullptr); 443 dispatch_block_t block = ^(void) { 444 handler(dispatch_get_context(source)); 445 }; 446 WRAP(dispatch_source_set_cancel_handler)(source, block); 447 } 448 449 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, 450 dispatch_source_t source, dispatch_block_t handler) { 451 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source, 452 handler); 453 if (handler == nullptr) 454 return REAL(dispatch_source_set_registration_handler)(source, nullptr); 455 dispatch_queue_t q = GetTargetQueueFromSource(source); 456 __block block_context_t new_context = { 457 q, handler, &invoke_block, false, false, false, 0}; 458 dispatch_block_t new_handler = Block_copy(^(void) { 459 new_context.orig_context = handler; // To explicitly capture "handler". 460 dispatch_callback_wrap(&new_context); 461 }); 462 uptr submit_sync = (uptr)&new_context; 463 Release(thr, pc, submit_sync); 464 REAL(dispatch_source_set_registration_handler)(source, new_handler); 465 Block_release(new_handler); 466 } 467 468 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f, 469 dispatch_source_t source, dispatch_function_t handler) { 470 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source, 471 handler); 472 if (handler == nullptr) 473 return REAL(dispatch_source_set_registration_handler)(source, nullptr); 474 dispatch_block_t block = ^(void) { 475 handler(dispatch_get_context(source)); 476 }; 477 WRAP(dispatch_source_set_registration_handler)(source, block); 478 } 479 480 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, 481 dispatch_queue_t queue, 482 DISPATCH_NOESCAPE void (^block)(size_t)) { 483 SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); 484 485 u8 sync1, sync2; 486 uptr parent_to_child_sync = (uptr)&sync1; 487 uptr child_to_parent_sync = (uptr)&sync2; 488 489 Release(thr, pc, parent_to_child_sync); 490 void (^new_block)(size_t) = ^(size_t iteration) { 491 SCOPED_INTERCEPTOR_RAW(dispatch_apply); 492 Acquire(thr, pc, parent_to_child_sync); 493 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 494 block(iteration); 495 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 496 Release(thr, pc, child_to_parent_sync); 497 }; 498 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 499 REAL(dispatch_apply)(iterations, queue, new_block); 500 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 501 Acquire(thr, pc, child_to_parent_sync); 502 } 503 504 static void invoke_block_iteration(void *param, size_t iteration) { 505 auto block = (void (^)(size_t)) param; 506 block(iteration); 507 } 508 509 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, 510 dispatch_queue_t queue, void *context, 511 void (*work)(void *, size_t)) { 512 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); 513 514 // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch 515 // implements dispatch_apply in terms of dispatch_apply_f. 516 u8 sync1, sync2; 517 uptr parent_to_child_sync = (uptr)&sync1; 518 uptr child_to_parent_sync = (uptr)&sync2; 519 520 Release(thr, pc, parent_to_child_sync); 521 void (^new_block)(size_t) = ^(size_t iteration) { 522 SCOPED_INTERCEPTOR_RAW(dispatch_apply_f); 523 Acquire(thr, pc, parent_to_child_sync); 524 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 525 work(context, iteration); 526 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 527 Release(thr, pc, child_to_parent_sync); 528 }; 529 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 530 REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration); 531 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 532 Acquire(thr, pc, child_to_parent_sync); 533 } 534 535 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) 536 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz) 537 538 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, 539 size_t size, dispatch_queue_t q, dispatch_block_t destructor) { 540 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor); 541 if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT)) 542 return REAL(dispatch_data_create)(buffer, size, q, destructor); 543 544 if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE) 545 destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); }; 546 else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP) 547 destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); }; 548 549 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 550 dispatch_block_t heap_block = Block_copy(destructor); 551 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 552 block_context_t *new_context = 553 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); 554 uptr submit_sync = (uptr)new_context; 555 Release(thr, pc, submit_sync); 556 return REAL(dispatch_data_create)(buffer, size, q, ^(void) { 557 dispatch_callback_wrap(new_context); 558 }); 559 } 560 561 typedef void (^fd_handler_t)(dispatch_data_t data, int error); 562 typedef void (^cleanup_handler_t)(int error); 563 564 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, 565 dispatch_queue_t q, fd_handler_t h) { 566 SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); 567 __block block_context_t new_context = { 568 q, nullptr, &invoke_block, false, false, false, 0}; 569 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { 570 new_context.orig_context = ^(void) { 571 h(data, error); 572 }; 573 dispatch_callback_wrap(&new_context); 574 }); 575 uptr submit_sync = (uptr)&new_context; 576 Release(thr, pc, submit_sync); 577 REAL(dispatch_read)(fd, length, q, new_h); 578 Block_release(new_h); 579 } 580 581 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, 582 dispatch_queue_t q, fd_handler_t h) { 583 SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); 584 __block block_context_t new_context = { 585 q, nullptr, &invoke_block, false, false, false, 0}; 586 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { 587 new_context.orig_context = ^(void) { 588 h(data, error); 589 }; 590 dispatch_callback_wrap(&new_context); 591 }); 592 uptr submit_sync = (uptr)&new_context; 593 Release(thr, pc, submit_sync); 594 REAL(dispatch_write)(fd, data, q, new_h); 595 Block_release(new_h); 596 } 597 598 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, 599 size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { 600 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); 601 __block block_context_t new_context = { 602 q, nullptr, &invoke_block, false, false, false, 0}; 603 dispatch_io_handler_t new_h = 604 Block_copy(^(bool done, dispatch_data_t data, int error) { 605 new_context.orig_context = ^(void) { 606 h(done, data, error); 607 }; 608 dispatch_callback_wrap(&new_context); 609 }); 610 uptr submit_sync = (uptr)&new_context; 611 Release(thr, pc, submit_sync); 612 REAL(dispatch_io_read)(channel, offset, length, q, new_h); 613 Block_release(new_h); 614 } 615 616 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, 617 dispatch_data_t data, dispatch_queue_t q, 618 dispatch_io_handler_t h) { 619 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); 620 __block block_context_t new_context = { 621 q, nullptr, &invoke_block, false, false, false, 0}; 622 dispatch_io_handler_t new_h = 623 Block_copy(^(bool done, dispatch_data_t data, int error) { 624 new_context.orig_context = ^(void) { 625 h(done, data, error); 626 }; 627 dispatch_callback_wrap(&new_context); 628 }); 629 uptr submit_sync = (uptr)&new_context; 630 Release(thr, pc, submit_sync); 631 REAL(dispatch_io_write)(channel, offset, data, q, new_h); 632 Block_release(new_h); 633 } 634 635 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, 636 dispatch_block_t barrier) { 637 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); 638 __block block_context_t new_context = { 639 nullptr, nullptr, &invoke_block, false, false, false, 0}; 640 new_context.non_queue_sync_object = (uptr)channel; 641 new_context.is_barrier_block = true; 642 dispatch_block_t new_block = Block_copy(^(void) { 643 new_context.orig_context = ^(void) { 644 barrier(); 645 }; 646 dispatch_callback_wrap(&new_context); 647 }); 648 uptr submit_sync = (uptr)&new_context; 649 Release(thr, pc, submit_sync); 650 REAL(dispatch_io_barrier)(channel, new_block); 651 Block_release(new_block); 652 } 653 654 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, 655 dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { 656 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); 657 __block dispatch_io_t new_channel = nullptr; 658 __block block_context_t new_context = { 659 q, nullptr, &invoke_block, false, false, false, 0}; 660 cleanup_handler_t new_h = Block_copy(^(int error) { 661 { 662 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 663 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 664 } 665 new_context.orig_context = ^(void) { 666 h(error); 667 }; 668 dispatch_callback_wrap(&new_context); 669 }); 670 uptr submit_sync = (uptr)&new_context; 671 Release(thr, pc, submit_sync); 672 new_channel = REAL(dispatch_io_create)(type, fd, q, new_h); 673 Block_release(new_h); 674 return new_channel; 675 } 676 677 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, 678 dispatch_io_type_t type, const char *path, int oflag, 679 mode_t mode, dispatch_queue_t q, cleanup_handler_t h) { 680 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, 681 q, h); 682 __block dispatch_io_t new_channel = nullptr; 683 __block block_context_t new_context = { 684 q, nullptr, &invoke_block, false, false, false, 0}; 685 cleanup_handler_t new_h = Block_copy(^(int error) { 686 { 687 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 688 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 689 } 690 new_context.orig_context = ^(void) { 691 h(error); 692 }; 693 dispatch_callback_wrap(&new_context); 694 }); 695 uptr submit_sync = (uptr)&new_context; 696 Release(thr, pc, submit_sync); 697 new_channel = 698 REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h); 699 Block_release(new_h); 700 return new_channel; 701 } 702 703 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, 704 dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q, 705 cleanup_handler_t h) { 706 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); 707 __block dispatch_io_t new_channel = nullptr; 708 __block block_context_t new_context = { 709 q, nullptr, &invoke_block, false, false, false, 0}; 710 cleanup_handler_t new_h = Block_copy(^(int error) { 711 { 712 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 713 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 714 } 715 new_context.orig_context = ^(void) { 716 h(error); 717 }; 718 dispatch_callback_wrap(&new_context); 719 }); 720 uptr submit_sync = (uptr)&new_context; 721 Release(thr, pc, submit_sync); 722 new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h); 723 Block_release(new_h); 724 return new_channel; 725 } 726 727 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel, 728 dispatch_io_close_flags_t flags) { 729 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags); 730 Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*]. 731 return REAL(dispatch_io_close)(channel, flags); 732 } 733 734 // Resuming a suspended queue needs to synchronize with all subsequent 735 // executions of blocks in that queue. 736 TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) { 737 SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o); 738 Release(thr, pc, (uptr)o); // Synchronizes with the Acquire() on serial_sync 739 // in dispatch_sync_pre_execute 740 return REAL(dispatch_resume)(o); 741 } 742 743 void InitializeLibdispatchInterceptors() { 744 INTERCEPT_FUNCTION(dispatch_async); 745 INTERCEPT_FUNCTION(dispatch_async_f); 746 INTERCEPT_FUNCTION(dispatch_sync); 747 INTERCEPT_FUNCTION(dispatch_sync_f); 748 INTERCEPT_FUNCTION(dispatch_barrier_async); 749 INTERCEPT_FUNCTION(dispatch_barrier_async_f); 750 INTERCEPT_FUNCTION(dispatch_barrier_sync); 751 INTERCEPT_FUNCTION(dispatch_barrier_sync_f); 752 INTERCEPT_FUNCTION(dispatch_async_and_wait); 753 INTERCEPT_FUNCTION(dispatch_async_and_wait_f); 754 INTERCEPT_FUNCTION(dispatch_after); 755 INTERCEPT_FUNCTION(dispatch_after_f); 756 INTERCEPT_FUNCTION(dispatch_once); 757 INTERCEPT_FUNCTION(dispatch_once_f); 758 INTERCEPT_FUNCTION(dispatch_semaphore_signal); 759 INTERCEPT_FUNCTION(dispatch_semaphore_wait); 760 INTERCEPT_FUNCTION(dispatch_group_wait); 761 INTERCEPT_FUNCTION(dispatch_group_leave); 762 INTERCEPT_FUNCTION(dispatch_group_async); 763 INTERCEPT_FUNCTION(dispatch_group_async_f); 764 INTERCEPT_FUNCTION(dispatch_group_notify); 765 INTERCEPT_FUNCTION(dispatch_group_notify_f); 766 INTERCEPT_FUNCTION(dispatch_source_set_event_handler); 767 INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f); 768 INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler); 769 INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f); 770 INTERCEPT_FUNCTION(dispatch_source_set_registration_handler); 771 INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f); 772 INTERCEPT_FUNCTION(dispatch_apply); 773 INTERCEPT_FUNCTION(dispatch_apply_f); 774 INTERCEPT_FUNCTION(dispatch_data_create); 775 INTERCEPT_FUNCTION(dispatch_read); 776 INTERCEPT_FUNCTION(dispatch_write); 777 INTERCEPT_FUNCTION(dispatch_io_read); 778 INTERCEPT_FUNCTION(dispatch_io_write); 779 INTERCEPT_FUNCTION(dispatch_io_barrier); 780 INTERCEPT_FUNCTION(dispatch_io_create); 781 INTERCEPT_FUNCTION(dispatch_io_create_with_path); 782 INTERCEPT_FUNCTION(dispatch_io_create_with_io); 783 INTERCEPT_FUNCTION(dispatch_io_close); 784 INTERCEPT_FUNCTION(dispatch_resume); 785 } 786 787 } // namespace __tsan 788