1 //===----------------------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include <memory> 10 11 #ifndef _LIBCPP_HAS_NO_THREADS 12 # include <mutex> 13 # include <thread> 14 # if defined(__ELF__) && defined(_LIBCPP_LINK_PTHREAD_LIB) 15 # pragma comment(lib, "pthread") 16 # endif 17 #endif 18 19 #include "include/atomic_support.h" 20 21 _LIBCPP_BEGIN_NAMESPACE_STD 22 23 const allocator_arg_t allocator_arg = allocator_arg_t(); 24 25 bad_weak_ptr::~bad_weak_ptr() noexcept {} 26 27 const char* 28 bad_weak_ptr::what() const noexcept 29 { 30 return "bad_weak_ptr"; 31 } 32 33 __shared_count::~__shared_count() 34 { 35 } 36 37 __shared_weak_count::~__shared_weak_count() 38 { 39 } 40 41 #if defined(_LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS) 42 void 43 __shared_count::__add_shared() noexcept 44 { 45 __libcpp_atomic_refcount_increment(__shared_owners_); 46 } 47 48 bool 49 __shared_count::__release_shared() noexcept 50 { 51 if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) 52 { 53 __on_zero_shared(); 54 return true; 55 } 56 return false; 57 } 58 59 void 60 __shared_weak_count::__add_shared() noexcept 61 { 62 __shared_count::__add_shared(); 63 } 64 65 void 66 __shared_weak_count::__add_weak() noexcept 67 { 68 __libcpp_atomic_refcount_increment(__shared_weak_owners_); 69 } 70 71 void 72 __shared_weak_count::__release_shared() noexcept 73 { 74 if (__shared_count::__release_shared()) 75 __release_weak(); 76 } 77 78 #endif // _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS 79 80 void 81 __shared_weak_count::__release_weak() noexcept 82 { 83 // NOTE: The acquire load here is an optimization of the very 84 // common case where a shared pointer is being destructed while 85 // having no other contended references. 86 // 87 // BENEFIT: We avoid expensive atomic stores like XADD and STREX 88 // in a common case. Those instructions are slow and do nasty 89 // things to caches. 90 // 91 // IS THIS SAFE? Yes. During weak destruction, if we see that we 92 // are the last reference, we know that no-one else is accessing 93 // us. If someone were accessing us, then they would be doing so 94 // while the last shared / weak_ptr was being destructed, and 95 // that's undefined anyway. 96 // 97 // If we see anything other than a 0, then we have possible 98 // contention, and need to use an atomicrmw primitive. 99 // The same arguments don't apply for increment, where it is legal 100 // (though inadvisable) to share shared_ptr references between 101 // threads, and have them all get copied at once. The argument 102 // also doesn't apply for __release_shared, because an outstanding 103 // weak_ptr::lock() could read / modify the shared count. 104 if (__libcpp_atomic_load(&__shared_weak_owners_, _AO_Acquire) == 0) 105 { 106 // no need to do this store, because we are about 107 // to destroy everything. 108 //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release); 109 __on_zero_shared_weak(); 110 } 111 else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1) 112 __on_zero_shared_weak(); 113 } 114 115 __shared_weak_count* 116 __shared_weak_count::lock() noexcept 117 { 118 long object_owners = __libcpp_atomic_load(&__shared_owners_); 119 while (object_owners != -1) 120 { 121 if (__libcpp_atomic_compare_exchange(&__shared_owners_, 122 &object_owners, 123 object_owners+1)) 124 return this; 125 } 126 return nullptr; 127 } 128 129 const void* 130 __shared_weak_count::__get_deleter(const type_info&) const noexcept 131 { 132 return nullptr; 133 } 134 135 #if !defined(_LIBCPP_HAS_NO_THREADS) 136 137 static constexpr std::size_t __sp_mut_count = 16; 138 static constinit __libcpp_mutex_t mut_back[__sp_mut_count] = 139 { 140 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 141 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 142 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, 143 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER 144 }; 145 146 _LIBCPP_CONSTEXPR __sp_mut::__sp_mut(void* p) noexcept 147 : __lx(p) 148 { 149 } 150 151 void 152 __sp_mut::lock() noexcept 153 { 154 auto m = static_cast<__libcpp_mutex_t*>(__lx); 155 unsigned count = 0; 156 while (!__libcpp_mutex_trylock(m)) 157 { 158 if (++count > 16) 159 { 160 __libcpp_mutex_lock(m); 161 break; 162 } 163 this_thread::yield(); 164 } 165 } 166 167 void 168 __sp_mut::unlock() noexcept 169 { 170 __libcpp_mutex_unlock(static_cast<__libcpp_mutex_t*>(__lx)); 171 } 172 173 __sp_mut& 174 __get_sp_mut(const void* p) 175 { 176 static constinit __sp_mut muts[__sp_mut_count] = { 177 &mut_back[ 0], &mut_back[ 1], &mut_back[ 2], &mut_back[ 3], 178 &mut_back[ 4], &mut_back[ 5], &mut_back[ 6], &mut_back[ 7], 179 &mut_back[ 8], &mut_back[ 9], &mut_back[10], &mut_back[11], 180 &mut_back[12], &mut_back[13], &mut_back[14], &mut_back[15] 181 }; 182 return muts[hash<const void*>()(p) & (__sp_mut_count-1)]; 183 } 184 185 #endif // !defined(_LIBCPP_HAS_NO_THREADS) 186 187 void* 188 align(size_t alignment, size_t size, void*& ptr, size_t& space) 189 { 190 void* r = nullptr; 191 if (size <= space) 192 { 193 char* p1 = static_cast<char*>(ptr); 194 char* p2 = reinterpret_cast<char*>(reinterpret_cast<size_t>(p1 + (alignment - 1)) & -alignment); 195 size_t d = static_cast<size_t>(p2 - p1); 196 if (d <= space - size) 197 { 198 r = p2; 199 ptr = r; 200 space -= d; 201 } 202 } 203 return r; 204 } 205 206 _LIBCPP_END_NAMESPACE_STD 207