1 //===-- scudo_tsd_shared.cpp ------------------------------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// Scudo shared TSD implementation.
11 ///
12 //===----------------------------------------------------------------------===//
13
14 #include "scudo_tsd.h"
15
16 #if !SCUDO_TSD_EXCLUSIVE
17
18 namespace __scudo {
19
20 static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT;
21 pthread_key_t PThreadKey;
22
23 static atomic_uint32_t CurrentIndex;
24 static ScudoTSD *TSDs;
25 static u32 NumberOfTSDs;
26 static u32 CoPrimes[SCUDO_SHARED_TSD_POOL_SIZE];
27 static u32 NumberOfCoPrimes = 0;
28
29 #if SANITIZER_LINUX && !SANITIZER_ANDROID
30 __attribute__((tls_model("initial-exec")))
31 THREADLOCAL ScudoTSD *CurrentTSD;
32 #endif
33
initOnce()34 static void initOnce() {
35 CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0);
36 initScudo();
37 NumberOfTSDs = Min(Max(1U, GetNumberOfCPUsCached()),
38 static_cast<u32>(SCUDO_SHARED_TSD_POOL_SIZE));
39 TSDs = reinterpret_cast<ScudoTSD *>(
40 MmapOrDie(sizeof(ScudoTSD) * NumberOfTSDs, "ScudoTSDs"));
41 for (u32 I = 0; I < NumberOfTSDs; I++) {
42 TSDs[I].init();
43 u32 A = I + 1;
44 u32 B = NumberOfTSDs;
45 while (B != 0) { const u32 T = A; A = B; B = T % B; }
46 if (A == 1)
47 CoPrimes[NumberOfCoPrimes++] = I + 1;
48 }
49 }
50
setCurrentTSD(ScudoTSD * TSD)51 ALWAYS_INLINE void setCurrentTSD(ScudoTSD *TSD) {
52 #if SANITIZER_ANDROID
53 *get_android_tls_ptr() = reinterpret_cast<uptr>(TSD);
54 #elif SANITIZER_LINUX
55 CurrentTSD = TSD;
56 #else
57 CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast<void *>(TSD)), 0);
58 #endif // SANITIZER_ANDROID
59 }
60
initThread(bool MinimalInit)61 void initThread(bool MinimalInit) {
62 pthread_once(&GlobalInitialized, initOnce);
63 // Initial context assignment is done in a plain round-robin fashion.
64 u32 Index = atomic_fetch_add(&CurrentIndex, 1, memory_order_relaxed);
65 setCurrentTSD(&TSDs[Index % NumberOfTSDs]);
66 }
67
getTSDAndLockSlow(ScudoTSD * TSD)68 ScudoTSD *getTSDAndLockSlow(ScudoTSD *TSD) {
69 if (NumberOfTSDs > 1) {
70 // Use the Precedence of the current TSD as our random seed. Since we are in
71 // the slow path, it means that tryLock failed, and as a result it's very
72 // likely that said Precedence is non-zero.
73 u32 RandState = static_cast<u32>(TSD->getPrecedence());
74 const u32 R = Rand(&RandState);
75 const u32 Inc = CoPrimes[R % NumberOfCoPrimes];
76 u32 Index = R % NumberOfTSDs;
77 uptr LowestPrecedence = UINTPTR_MAX;
78 ScudoTSD *CandidateTSD = nullptr;
79 // Go randomly through at most 4 contexts and find a candidate.
80 for (u32 I = 0; I < Min(4U, NumberOfTSDs); I++) {
81 if (TSDs[Index].tryLock()) {
82 setCurrentTSD(&TSDs[Index]);
83 return &TSDs[Index];
84 }
85 const uptr Precedence = TSDs[Index].getPrecedence();
86 // A 0 precedence here means another thread just locked this TSD.
87 if (UNLIKELY(Precedence == 0))
88 continue;
89 if (Precedence < LowestPrecedence) {
90 CandidateTSD = &TSDs[Index];
91 LowestPrecedence = Precedence;
92 }
93 Index += Inc;
94 if (Index >= NumberOfTSDs)
95 Index -= NumberOfTSDs;
96 }
97 if (CandidateTSD) {
98 CandidateTSD->lock();
99 setCurrentTSD(CandidateTSD);
100 return CandidateTSD;
101 }
102 }
103 // Last resort, stick with the current one.
104 TSD->lock();
105 return TSD;
106 }
107
108 } // namespace __scudo
109
110 #endif // !SCUDO_TSD_EXCLUSIVE
111