?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Run DCE after a LoopFlatten test to reduce spurious output [nfc]
[llvm-project.git] / compiler-rt / lib / tsan / rtl / tsan_interceptors_libdispatch.cpp
blob2104fe7fd059f53ff4a6b3cbe0e56890aefa4cc9
1 //===-- tsan_interceptors_libdispatch.cpp ---------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 //
11 // Support for intercepting libdispatch (GCD).
12 //===----------------------------------------------------------------------===//
13
14 #include "sanitizer_common/sanitizer_common.h"
15 #include "interception/interception.h"
16 #include "tsan_interceptors.h"
17 #include "tsan_rtl.h"
18
19 #include "BlocksRuntime/Block.h"
20 #include "tsan_dispatch_defs.h"
21
22 #if SANITIZER_APPLE
23 # include <Availability.h>
24 #endif
25
26 namespace __tsan {
27 typedef u16 uint16_t;
28
29 typedef struct {
30 dispatch_queue_t queue;
31 void *orig_context;
32 dispatch_function_t orig_work;
33 bool free_context_in_callback;
34 bool submitted_synchronously;
35 bool is_barrier_block;
36 uptr non_queue_sync_object;
37 } block_context_t;
38
39 // The offsets of different fields of the dispatch_queue_t structure, exported
40 // by libdispatch.dylib.
41 extern "C" struct dispatch_queue_offsets_s {
42 const uint16_t dqo_version;
43 const uint16_t dqo_label;
44 const uint16_t dqo_label_size;
45 const uint16_t dqo_flags;
46 const uint16_t dqo_flags_size;
47 const uint16_t dqo_serialnum;
48 const uint16_t dqo_serialnum_size;
49 const uint16_t dqo_width;
50 const uint16_t dqo_width_size;
51 const uint16_t dqo_running;
52 const uint16_t dqo_running_size;
53 const uint16_t dqo_suspend_cnt;
54 const uint16_t dqo_suspend_cnt_size;
55 const uint16_t dqo_target_queue;
56 const uint16_t dqo_target_queue_size;
57 const uint16_t dqo_priority;
58 const uint16_t dqo_priority_size;
59 } dispatch_queue_offsets;
60
61 static bool IsQueueSerial(dispatch_queue_t q) {
62 CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
63 uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
64 CHECK_NE(width, 0);
65 return width == 1;
66 }
67
68 static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) {
69 CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
70 dispatch_queue_t tq = *(
71 dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue);
72 return tq;
73 }
74
75 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
76 dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source);
77 CHECK_NE(tq, 0);
78 return tq;
79 }
80
81 static block_context_t *AllocContext(ThreadState *thr, uptr pc,
82 dispatch_queue_t queue, void *orig_context,
83 dispatch_function_t orig_work) {
84 block_context_t *new_context =
85 (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t));
86 new_context->queue = queue;
87 new_context->orig_context = orig_context;
88 new_context->orig_work = orig_work;
89 new_context->free_context_in_callback = true;
90 new_context->submitted_synchronously = false;
91 new_context->is_barrier_block = false;
92 new_context->non_queue_sync_object = 0;
93 return new_context;
94 }
95
96 #define GET_QUEUE_SYNC_VARS(context, q) \
97 bool is_queue_serial = q && IsQueueSerial(q); \
98 uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \
99 uptr serial_sync = (uptr)sync_ptr; \
100 uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \
101 bool serial_task = context->is_barrier_block || is_queue_serial
102
103 static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc,
104 block_context_t *context) {
105 uptr submit_sync = (uptr)context;
106 Acquire(thr, pc, submit_sync);
107
108 dispatch_queue_t q = context->queue;
109 do {
110 GET_QUEUE_SYNC_VARS(context, q);
111 if (serial_sync) Acquire(thr, pc, serial_sync);
112 if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync);
113
114 if (q) q = GetTargetQueueFromQueue(q);
115 } while (q);
116 }
117
118 static void dispatch_sync_post_execute(ThreadState *thr, uptr pc,
119 block_context_t *context) {
120 uptr submit_sync = (uptr)context;
121 if (context->submitted_synchronously) Release(thr, pc, submit_sync);
122
123 dispatch_queue_t q = context->queue;
124 do {
125 GET_QUEUE_SYNC_VARS(context, q);
126 if (serial_task && serial_sync) Release(thr, pc, serial_sync);
127 if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync);
128
129 if (q) q = GetTargetQueueFromQueue(q);
130 } while (q);
131 }
132
133 static void dispatch_callback_wrap(void *param) {
134 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
135 block_context_t *context = (block_context_t *)param;
136
137 dispatch_sync_pre_execute(thr, pc, context);
138
139 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
140 context->orig_work(context->orig_context);
141 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
142
143 dispatch_sync_post_execute(thr, pc, context);
144
145 if (context->free_context_in_callback) user_free(thr, pc, context);
146 }
147
148 static void invoke_block(void *param) {
149 dispatch_block_t block = (dispatch_block_t)param;
150 block();
151 }
152
153 static void invoke_and_release_block(void *param) {
154 dispatch_block_t block = (dispatch_block_t)param;
155 block();
156 Block_release(block);
157 }
158
159 #define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \
160 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
161 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
162 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
163 dispatch_block_t heap_block = Block_copy(block); \
164 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
165 block_context_t *new_context = \
166 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \
167 new_context->is_barrier_block = barrier; \
168 Release(thr, pc, (uptr)new_context); \
169 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
170 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \
171 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
172 }
173
174 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \
175 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \
176 DISPATCH_NOESCAPE dispatch_block_t block) { \
177 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
178 block_context_t new_context = { \
179 q, block, &invoke_block, false, true, barrier, 0}; \
180 Release(thr, pc, (uptr)&new_context); \
181 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
182 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \
183 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
184 Acquire(thr, pc, (uptr)&new_context); \
185 }
186
187 #define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \
188 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
189 dispatch_function_t work) { \
190 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
191 block_context_t *new_context = \
192 AllocContext(thr, pc, q, context, work); \
193 new_context->is_barrier_block = barrier; \
194 Release(thr, pc, (uptr)new_context); \
195 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
196 REAL(name)(q, new_context, dispatch_callback_wrap); \
197 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
198 }
199
200 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \
201 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
202 dispatch_function_t work) { \
203 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
204 block_context_t new_context = { \
205 q, context, work, false, true, barrier, 0}; \
206 Release(thr, pc, (uptr)&new_context); \
207 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
208 REAL(name)(q, &new_context, dispatch_callback_wrap); \
209 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
210 Acquire(thr, pc, (uptr)&new_context); \
211 }
212
213 #define DISPATCH_INTERCEPT(name, barrier) \
214 DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \
215 DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \
216 DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \
217 DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier)
218
219 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new
220 // context, which is used to synchronize (we release the context before
221 // submitting, and the callback acquires it before executing the original
222 // callback).
223 DISPATCH_INTERCEPT(dispatch, false)
224 DISPATCH_INTERCEPT(dispatch_barrier, true)
225
226 // dispatch_async_and_wait() and friends were introduced in macOS 10.14.
227 // Linking of these interceptors fails when using an older SDK.
228 #if !SANITIZER_APPLE || defined(__MAC_10_14)
229 // macOS 10.14 is greater than our minimal deployment target. To ensure we
230 // generate a weak reference so the TSan dylib continues to work on older
231 // systems, we need to forward declare the intercepted functions as "weak
232 // imports". Note that this file is multi-platform, so we cannot include the
233 // actual header file (#include <dispatch/dispatch.h>).
234 SANITIZER_WEAK_IMPORT void dispatch_async_and_wait(
235 dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block);
236 SANITIZER_WEAK_IMPORT void dispatch_async_and_wait_f(
237 dispatch_queue_t queue, void *context, dispatch_function_t work);
238 SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait(
239 dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block);
240 SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait_f(
241 dispatch_queue_t queue, void *context, dispatch_function_t work);
242
243 DISPATCH_INTERCEPT_SYNC_F(dispatch_async_and_wait_f, false)
244 DISPATCH_INTERCEPT_SYNC_B(dispatch_async_and_wait, false)
245 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_async_and_wait_f, true)
246 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_async_and_wait, true)
247 #endif
248
249
250 DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when,
251 dispatch_queue_t queue, void *context, dispatch_function_t work)
252
253 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
254 dispatch_queue_t queue, dispatch_block_t block) {
255 SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
256 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
257 dispatch_block_t heap_block = Block_copy(block);
258 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
259 block_context_t *new_context =
260 AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
261 Release(thr, pc, (uptr)new_context);
262 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
263 REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
264 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
265 }
266
267 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
268 dispatch_queue_t queue, void *context,
269 dispatch_function_t work) {
270 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
271 WRAP(dispatch_after)(when, queue, ^(void) {
272 work(context);
273 });
274 }
275
276 // GCD's dispatch_once implementation has a fast path that contains a racy read
277 // and it's inlined into user's code. Furthermore, this fast path doesn't
278 // establish a proper happens-before relations between the initialization and
279 // code following the call to dispatch_once. We could deal with this in
280 // instrumented code, but there's not much we can do about it in system
281 // libraries. Let's disable the fast path (by never storing the value ~0 to
282 // predicate), so the interceptor is always called, and let's add proper release
283 // and acquire semantics. Since TSan does not see its own atomic stores, the
284 // race on predicate won't be reported - the only accesses to it that TSan sees
285 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
286 // both a macro and a real function, we want to intercept the function, so we
287 // need to undefine the macro.
288 #undef dispatch_once
289 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
290 DISPATCH_NOESCAPE dispatch_block_t block) {
291 SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
292 atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
293 u32 v = atomic_load(a, memory_order_acquire);
294 if (v == 0 &&
295 atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
296 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
297 block();
298 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
299 Release(thr, pc, (uptr)a);
300 atomic_store(a, 2, memory_order_release);
301 } else {
302 while (v != 2) {
303 internal_sched_yield();
304 v = atomic_load(a, memory_order_acquire);
305 }
306 Acquire(thr, pc, (uptr)a);
307 }
308 }
309
310 #undef dispatch_once_f
311 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
312 void *context, dispatch_function_t function) {
313 SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
314 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
315 WRAP(dispatch_once)(predicate, ^(void) {
316 function(context);
317 });
318 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
319 }
320
321 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
322 dispatch_semaphore_t dsema) {
323 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
324 Release(thr, pc, (uptr)dsema);
325 return REAL(dispatch_semaphore_signal)(dsema);
326 }
327
328 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
329 dispatch_time_t timeout) {
330 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
331 long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
332 if (result == 0) Acquire(thr, pc, (uptr)dsema);
333 return result;
334 }
335
336 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
337 dispatch_time_t timeout) {
338 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
339 long_t result = REAL(dispatch_group_wait)(group, timeout);
340 if (result == 0) Acquire(thr, pc, (uptr)group);
341 return result;
342 }
343
344 // Used, but not intercepted.
345 extern "C" void dispatch_group_enter(dispatch_group_t group);
346
347 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
348 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
349 // Acquired in the group notification callback in dispatch_group_notify[_f].
350 Release(thr, pc, (uptr)group);
351 REAL(dispatch_group_leave)(group);
352 }
353
354 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
355 dispatch_queue_t queue, dispatch_block_t block) {
356 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
357 dispatch_retain(group);
358 dispatch_group_enter(group);
359 __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block);
360 WRAP(dispatch_async)(queue, ^(void) {
361 block_copy();
362 Block_release(block_copy);
363 WRAP(dispatch_group_leave)(group);
364 dispatch_release(group);
365 });
366 }
367
368 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
369 dispatch_queue_t queue, void *context,
370 dispatch_function_t work) {
371 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
372 dispatch_retain(group);
373 dispatch_group_enter(group);
374 WRAP(dispatch_async)(queue, ^(void) {
375 work(context);
376 WRAP(dispatch_group_leave)(group);
377 dispatch_release(group);
378 });
379 }
380
381 DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group,
382 dispatch_queue_t q, void *context, dispatch_function_t work)
383
384 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
385 dispatch_queue_t q, dispatch_block_t block) {
386 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
387
388 // To make sure the group is still available in the callback (otherwise
389 // it can be already destroyed). Will be released in the callback.
390 dispatch_retain(group);
391
392 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
393 dispatch_block_t heap_block = Block_copy(^(void) {
394 {
395 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
396 // Released when leaving the group (dispatch_group_leave).
397 Acquire(thr, pc, (uptr)group);
398 }
399 dispatch_release(group);
400 block();
401 });
402 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
403 block_context_t *new_context =
404 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
405 new_context->is_barrier_block = true;
406 Release(thr, pc, (uptr)new_context);
407 REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
408 }
409
410 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
411 dispatch_queue_t q, void *context, dispatch_function_t work) {
412 WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
413 }
414
415 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
416 dispatch_source_t source, dispatch_block_t handler) {
417 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
418 if (handler == nullptr)
419 return REAL(dispatch_source_set_event_handler)(source, nullptr);
420 dispatch_queue_t q = GetTargetQueueFromSource(source);
421 __block block_context_t new_context = {
422 q, handler, &invoke_block, false, false, false, 0 };
423 dispatch_block_t new_handler = Block_copy(^(void) {
424 new_context.orig_context = handler; // To explicitly capture "handler".
425 dispatch_callback_wrap(&new_context);
426 });
427 uptr submit_sync = (uptr)&new_context;
428 Release(thr, pc, submit_sync);
429 REAL(dispatch_source_set_event_handler)(source, new_handler);
430 Block_release(new_handler);
431 }
432
433 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
434 dispatch_source_t source, dispatch_function_t handler) {
435 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
436 if (handler == nullptr)
437 return REAL(dispatch_source_set_event_handler)(source, nullptr);
438 dispatch_block_t block = ^(void) {
439 handler(dispatch_get_context(source));
440 };
441 WRAP(dispatch_source_set_event_handler)(source, block);
442 }
443
444 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
445 dispatch_source_t source, dispatch_block_t handler) {
446 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
447 if (handler == nullptr)
448 return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
449 dispatch_queue_t q = GetTargetQueueFromSource(source);
450 __block block_context_t new_context = {
451 q, handler, &invoke_block, false, false, false, 0};
452 dispatch_block_t new_handler = Block_copy(^(void) {
453 new_context.orig_context = handler; // To explicitly capture "handler".
454 dispatch_callback_wrap(&new_context);
455 });
456 uptr submit_sync = (uptr)&new_context;
457 Release(thr, pc, submit_sync);
458 REAL(dispatch_source_set_cancel_handler)(source, new_handler);
459 Block_release(new_handler);
460 }
461
462 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
463 dispatch_source_t source, dispatch_function_t handler) {
464 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
465 handler);
466 if (handler == nullptr)
467 return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
468 dispatch_block_t block = ^(void) {
469 handler(dispatch_get_context(source));
470 };
471 WRAP(dispatch_source_set_cancel_handler)(source, block);
472 }
473
474 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
475 dispatch_source_t source, dispatch_block_t handler) {
476 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
477 handler);
478 if (handler == nullptr)
479 return REAL(dispatch_source_set_registration_handler)(source, nullptr);
480 dispatch_queue_t q = GetTargetQueueFromSource(source);
481 __block block_context_t new_context = {
482 q, handler, &invoke_block, false, false, false, 0};
483 dispatch_block_t new_handler = Block_copy(^(void) {
484 new_context.orig_context = handler; // To explicitly capture "handler".
485 dispatch_callback_wrap(&new_context);
486 });
487 uptr submit_sync = (uptr)&new_context;
488 Release(thr, pc, submit_sync);
489 REAL(dispatch_source_set_registration_handler)(source, new_handler);
490 Block_release(new_handler);
491 }
492
493 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
494 dispatch_source_t source, dispatch_function_t handler) {
495 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
496 handler);
497 if (handler == nullptr)
498 return REAL(dispatch_source_set_registration_handler)(source, nullptr);
499 dispatch_block_t block = ^(void) {
500 handler(dispatch_get_context(source));
501 };
502 WRAP(dispatch_source_set_registration_handler)(source, block);
503 }
504
505 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
506 dispatch_queue_t queue,
507 DISPATCH_NOESCAPE void (^block)(size_t)) {
508 SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
509
510 u8 sync1, sync2;
511 uptr parent_to_child_sync = (uptr)&sync1;
512 uptr child_to_parent_sync = (uptr)&sync2;
513
514 Release(thr, pc, parent_to_child_sync);
515 void (^new_block)(size_t) = ^(size_t iteration) {
516 SCOPED_INTERCEPTOR_RAW(dispatch_apply);
517 Acquire(thr, pc, parent_to_child_sync);
518 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
519 block(iteration);
520 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
521 Release(thr, pc, child_to_parent_sync);
522 };
523 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
524 REAL(dispatch_apply)(iterations, queue, new_block);
525 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
526 Acquire(thr, pc, child_to_parent_sync);
527 }
528
529 static void invoke_block_iteration(void *param, size_t iteration) {
530 auto block = (void (^)(size_t)) param;
531 block(iteration);
532 }
533
534 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
535 dispatch_queue_t queue, void *context,
536 void (*work)(void *, size_t)) {
537 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
538
539 // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch
540 // implements dispatch_apply in terms of dispatch_apply_f.
541 u8 sync1, sync2;
542 uptr parent_to_child_sync = (uptr)&sync1;
543 uptr child_to_parent_sync = (uptr)&sync2;
544
545 Release(thr, pc, parent_to_child_sync);
546 void (^new_block)(size_t) = ^(size_t iteration) {
547 SCOPED_INTERCEPTOR_RAW(dispatch_apply_f);
548 Acquire(thr, pc, parent_to_child_sync);
549 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
550 work(context, iteration);
551 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
552 Release(thr, pc, child_to_parent_sync);
553 };
554 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
555 REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration);
556 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
557 Acquire(thr, pc, child_to_parent_sync);
558 }
559
560 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
561 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, SIZE_T sz)
562
563 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
564 size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
565 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
566 if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
567 return REAL(dispatch_data_create)(buffer, size, q, destructor);
568
569 if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
570 destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); };
571 else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
572 destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); };
573
574 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
575 dispatch_block_t heap_block = Block_copy(destructor);
576 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
577 block_context_t *new_context =
578 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
579 uptr submit_sync = (uptr)new_context;
580 Release(thr, pc, submit_sync);
581 return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
582 dispatch_callback_wrap(new_context);
583 });
584 }
585
586 typedef void (^fd_handler_t)(dispatch_data_t data, int error);
587 typedef void (^cleanup_handler_t)(int error);
588
589 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
590 dispatch_queue_t q, fd_handler_t h) {
591 SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
592 __block block_context_t new_context = {
593 q, nullptr, &invoke_block, false, false, false, 0};
594 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
595 new_context.orig_context = ^(void) {
596 h(data, error);
597 };
598 dispatch_callback_wrap(&new_context);
599 });
600 uptr submit_sync = (uptr)&new_context;
601 Release(thr, pc, submit_sync);
602 REAL(dispatch_read)(fd, length, q, new_h);
603 Block_release(new_h);
604 }
605
606 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
607 dispatch_queue_t q, fd_handler_t h) {
608 SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
609 __block block_context_t new_context = {
610 q, nullptr, &invoke_block, false, false, false, 0};
611 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
612 new_context.orig_context = ^(void) {
613 h(data, error);
614 };
615 dispatch_callback_wrap(&new_context);
616 });
617 uptr submit_sync = (uptr)&new_context;
618 Release(thr, pc, submit_sync);
619 REAL(dispatch_write)(fd, data, q, new_h);
620 Block_release(new_h);
621 }
622
623 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
624 size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
625 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
626 __block block_context_t new_context = {
627 q, nullptr, &invoke_block, false, false, false, 0};
628 dispatch_io_handler_t new_h =
629 Block_copy(^(bool done, dispatch_data_t data, int error) {
630 new_context.orig_context = ^(void) {
631 h(done, data, error);
632 };
633 dispatch_callback_wrap(&new_context);
634 });
635 uptr submit_sync = (uptr)&new_context;
636 Release(thr, pc, submit_sync);
637 REAL(dispatch_io_read)(channel, offset, length, q, new_h);
638 Block_release(new_h);
639 }
640
641 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
642 dispatch_data_t data, dispatch_queue_t q,
643 dispatch_io_handler_t h) {
644 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
645 __block block_context_t new_context = {
646 q, nullptr, &invoke_block, false, false, false, 0};
647 dispatch_io_handler_t new_h =
648 Block_copy(^(bool done, dispatch_data_t data, int error) {
649 new_context.orig_context = ^(void) {
650 h(done, data, error);
651 };
652 dispatch_callback_wrap(&new_context);
653 });
654 uptr submit_sync = (uptr)&new_context;
655 Release(thr, pc, submit_sync);
656 REAL(dispatch_io_write)(channel, offset, data, q, new_h);
657 Block_release(new_h);
658 }
659
660 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
661 dispatch_block_t barrier) {
662 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
663 __block block_context_t new_context = {
664 nullptr, nullptr, &invoke_block, false, false, false, 0};
665 new_context.non_queue_sync_object = (uptr)channel;
666 new_context.is_barrier_block = true;
667 dispatch_block_t new_block = Block_copy(^(void) {
668 new_context.orig_context = ^(void) {
669 barrier();
670 };
671 dispatch_callback_wrap(&new_context);
672 });
673 uptr submit_sync = (uptr)&new_context;
674 Release(thr, pc, submit_sync);
675 REAL(dispatch_io_barrier)(channel, new_block);
676 Block_release(new_block);
677 }
678
679 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
680 dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
681 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
682 __block dispatch_io_t new_channel = nullptr;
683 __block block_context_t new_context = {
684 q, nullptr, &invoke_block, false, false, false, 0};
685 cleanup_handler_t new_h = Block_copy(^(int error) {
686 {
687 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
688 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
689 }
690 new_context.orig_context = ^(void) {
691 h(error);
692 };
693 dispatch_callback_wrap(&new_context);
694 });
695 uptr submit_sync = (uptr)&new_context;
696 Release(thr, pc, submit_sync);
697 new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
698 Block_release(new_h);
699 return new_channel;
700 }
701
702 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
703 dispatch_io_type_t type, const char *path, int oflag,
704 mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
705 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
706 q, h);
707 __block dispatch_io_t new_channel = nullptr;
708 __block block_context_t new_context = {
709 q, nullptr, &invoke_block, false, false, false, 0};
710 cleanup_handler_t new_h = Block_copy(^(int error) {
711 {
712 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
713 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
714 }
715 new_context.orig_context = ^(void) {
716 h(error);
717 };
718 dispatch_callback_wrap(&new_context);
719 });
720 uptr submit_sync = (uptr)&new_context;
721 Release(thr, pc, submit_sync);
722 new_channel =
723 REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
724 Block_release(new_h);
725 return new_channel;
726 }
727
728 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
729 dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
730 cleanup_handler_t h) {
731 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
732 __block dispatch_io_t new_channel = nullptr;
733 __block block_context_t new_context = {
734 q, nullptr, &invoke_block, false, false, false, 0};
735 cleanup_handler_t new_h = Block_copy(^(int error) {
736 {
737 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
738 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
739 }
740 new_context.orig_context = ^(void) {
741 h(error);
742 };
743 dispatch_callback_wrap(&new_context);
744 });
745 uptr submit_sync = (uptr)&new_context;
746 Release(thr, pc, submit_sync);
747 new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
748 Block_release(new_h);
749 return new_channel;
750 }
751
752 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
753 dispatch_io_close_flags_t flags) {
754 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
755 Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*].
756 return REAL(dispatch_io_close)(channel, flags);
757 }
758
759 // Resuming a suspended queue needs to synchronize with all subsequent
760 // executions of blocks in that queue.
761 TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) {
762 SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o);
763 Release(thr, pc, (uptr)o); // Synchronizes with the Acquire() on serial_sync
764 // in dispatch_sync_pre_execute
765 return REAL(dispatch_resume)(o);
766 }
767
768 void InitializeLibdispatchInterceptors() {
769 INTERCEPT_FUNCTION(dispatch_async);
770 INTERCEPT_FUNCTION(dispatch_async_f);
771 INTERCEPT_FUNCTION(dispatch_sync);
772 INTERCEPT_FUNCTION(dispatch_sync_f);
773 INTERCEPT_FUNCTION(dispatch_barrier_async);
774 INTERCEPT_FUNCTION(dispatch_barrier_async_f);
775 INTERCEPT_FUNCTION(dispatch_barrier_sync);
776 INTERCEPT_FUNCTION(dispatch_barrier_sync_f);
777 INTERCEPT_FUNCTION(dispatch_async_and_wait);
778 INTERCEPT_FUNCTION(dispatch_async_and_wait_f);
779 INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait);
780 INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait_f);
781 INTERCEPT_FUNCTION(dispatch_after);
782 INTERCEPT_FUNCTION(dispatch_after_f);
783 INTERCEPT_FUNCTION(dispatch_once);
784 INTERCEPT_FUNCTION(dispatch_once_f);
785 INTERCEPT_FUNCTION(dispatch_semaphore_signal);
786 INTERCEPT_FUNCTION(dispatch_semaphore_wait);
787 INTERCEPT_FUNCTION(dispatch_group_wait);
788 INTERCEPT_FUNCTION(dispatch_group_leave);
789 INTERCEPT_FUNCTION(dispatch_group_async);
790 INTERCEPT_FUNCTION(dispatch_group_async_f);
791 INTERCEPT_FUNCTION(dispatch_group_notify);
792 INTERCEPT_FUNCTION(dispatch_group_notify_f);
793 INTERCEPT_FUNCTION(dispatch_source_set_event_handler);
794 INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f);
795 INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler);
796 INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f);
797 INTERCEPT_FUNCTION(dispatch_source_set_registration_handler);
798 INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f);
799 INTERCEPT_FUNCTION(dispatch_apply);
800 INTERCEPT_FUNCTION(dispatch_apply_f);
801 INTERCEPT_FUNCTION(dispatch_data_create);
802 INTERCEPT_FUNCTION(dispatch_read);
803 INTERCEPT_FUNCTION(dispatch_write);
804 INTERCEPT_FUNCTION(dispatch_io_read);
805 INTERCEPT_FUNCTION(dispatch_io_write);
806 INTERCEPT_FUNCTION(dispatch_io_barrier);
807 INTERCEPT_FUNCTION(dispatch_io_create);
808 INTERCEPT_FUNCTION(dispatch_io_create_with_path);
809 INTERCEPT_FUNCTION(dispatch_io_create_with_io);
810 INTERCEPT_FUNCTION(dispatch_io_close);
811 INTERCEPT_FUNCTION(dispatch_resume);
812 }
813
814 } // namespace __tsan
LLVM monorepo