1 /* 2 Copyright (c) 2021 Intel Corporation 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 #ifndef __TBB_rw_mutex_H 18 #define __TBB_rw_mutex_H 19 20 #if __TBB_PREVIEW_MUTEXES 21 22 #include "detail/_namespace_injection.h" 23 #include "detail/_utils.h" 24 #include "detail/_waitable_atomic.h" 25 #include "detail/_scoped_lock.h" 26 #include "detail/_mutex_common.h" 27 #include "profiling.h" 28 29 namespace tbb { 30 namespace detail { 31 namespace d1 { 32 33 class rw_mutex { 34 public: 35 //! Constructors 36 rw_mutex() noexcept : m_state(0) { 37 create_itt_sync(this, "tbb::rw_mutex", ""); 38 } 39 40 //! Destructor 41 ~rw_mutex() { 42 __TBB_ASSERT(!m_state.load(std::memory_order_relaxed), "destruction of an acquired mutex"); 43 } 44 45 //! No Copy 46 rw_mutex(const rw_mutex&) = delete; 47 rw_mutex& operator=(const rw_mutex&) = delete; 48 49 using scoped_lock = rw_scoped_lock<rw_mutex>; 50 51 //! Mutex traits 52 static constexpr bool is_rw_mutex = true; 53 static constexpr bool is_recursive_mutex = false; 54 static constexpr bool is_fair_mutex = false; 55 56 //! Acquire lock 57 void lock() { 58 call_itt_notify(prepare, this); 59 while (!try_lock()) { 60 if (!(m_state.load(std::memory_order_relaxed) & WRITER_PENDING)) { // no pending writers 61 m_state |= WRITER_PENDING; 62 } 63 64 auto wakeup_condition = [&] { return !(m_state.load(std::memory_order_relaxed) & BUSY); }; 65 adaptive_wait_on_address(this, wakeup_condition, WRITER_CONTEXT); 66 } 67 68 call_itt_notify(acquired, this); 69 } 70 71 //! Try acquiring lock (non-blocking) 72 /** Return true if lock acquired; false otherwise. */ 73 bool try_lock() { 74 // for a writer: only possible to acquire if no active readers or writers 75 // Use relaxed memory fence is OK here because 76 // Acquire memory fence guaranteed by compare_exchange_strong() 77 state_type s = m_state.load(std::memory_order_relaxed); 78 if (!(s & BUSY)) { // no readers, no writers; mask is 1..1101 79 if (m_state.compare_exchange_strong(s, WRITER)) { 80 call_itt_notify(acquired, this); 81 return true; // successfully stored writer flag 82 } 83 } 84 return false; 85 } 86 87 //! Release lock 88 void unlock() { 89 call_itt_notify(releasing, this); 90 state_type curr_state = (m_state &= READERS | WRITER_PENDING); // Returns current state 91 92 if (curr_state & WRITER_PENDING) { 93 r1::notify_by_address(this, WRITER_CONTEXT); 94 } else { 95 // It's possible that WRITER sleeps without WRITER_PENDING, 96 // because other thread might clear this bit at upgrade() 97 r1::notify_by_address_all(this); 98 } 99 } 100 101 //! Lock shared ownership mutex 102 void lock_shared() { 103 call_itt_notify(prepare, this); 104 while (!try_lock_shared()) { 105 state_type has_writer = WRITER | WRITER_PENDING; 106 auto wakeup_condition = [&] { return !(m_state.load(std::memory_order_relaxed) & has_writer); }; 107 adaptive_wait_on_address(this, wakeup_condition, READER_CONTEXT); 108 } 109 __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state of a read lock: no readers"); 110 } 111 112 //! Try lock shared ownership mutex 113 bool try_lock_shared() { 114 // for a reader: acquire if no active or waiting writers 115 // Use relaxed memory fence is OK here because 116 // Acquire memory fence guaranteed by fetch_add() 117 state_type has_writer = WRITER | WRITER_PENDING; 118 if (!(m_state.load(std::memory_order_relaxed) & has_writer)) { 119 if (m_state.fetch_add(ONE_READER) & has_writer) { 120 m_state -= ONE_READER; 121 r1::notify_by_address(this, WRITER_CONTEXT); 122 } else { 123 call_itt_notify(acquired, this); 124 return true; // successfully stored increased number of readers 125 } 126 } 127 return false; 128 } 129 130 //! Unlock shared ownership mutex 131 void unlock_shared() { 132 __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state of a read lock: no readers"); 133 call_itt_notify(releasing, this); 134 135 state_type curr_state = (m_state -= ONE_READER); // Returns current state 136 137 if (curr_state & (WRITER_PENDING)) { 138 r1::notify_by_address(this, WRITER_CONTEXT); 139 } else { 140 // It's possible that WRITER sleeps without WRITER_PENDING, 141 // because other thread might clear this bit at upgrade() 142 r1::notify_by_address_all(this); 143 } 144 } 145 146 private: 147 /** Internal non ISO C++ standard API **/ 148 //! This API is used through the scoped_lock class 149 150 //! Upgrade reader to become a writer. 151 /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ 152 bool upgrade() { 153 state_type s = m_state.load(std::memory_order_relaxed); 154 __TBB_ASSERT(s & READERS, "invalid state before upgrade: no readers "); 155 // Check and set writer-pending flag. 156 // Required conditions: either no pending writers, or we are the only reader 157 // (with multiple readers and pending writer, another upgrade could have been requested) 158 while ((s & READERS) == ONE_READER || !(s & WRITER_PENDING)) { 159 if (m_state.compare_exchange_strong(s, s | WRITER | WRITER_PENDING)) { 160 auto wakeup_condition = [&] { return (m_state.load(std::memory_order_relaxed) & READERS) == ONE_READER; }; 161 while ((m_state.load(std::memory_order_relaxed) & READERS) != ONE_READER) { 162 adaptive_wait_on_address(this, wakeup_condition, WRITER_CONTEXT); 163 } 164 165 __TBB_ASSERT((m_state.load(std::memory_order_relaxed) & (WRITER_PENDING|WRITER)) == (WRITER_PENDING | WRITER), 166 "invalid state when upgrading to writer"); 167 // Both new readers and writers are blocked at this time 168 m_state -= (ONE_READER + WRITER_PENDING); 169 return true; // successfully upgraded 170 } 171 } 172 // Slow reacquire 173 unlock_shared(); 174 lock(); 175 return false; 176 } 177 178 //! Downgrade writer to a reader 179 void downgrade() { 180 __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & WRITER, nullptr), 181 call_itt_notify(releasing, this); 182 m_state += (ONE_READER - WRITER); 183 184 if (!(m_state & WRITER_PENDING)) { 185 r1::notify_by_address(this, READER_CONTEXT); 186 } 187 188 __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state after downgrade: no readers"); 189 } 190 191 using state_type = std::intptr_t; 192 static constexpr state_type WRITER = 1; 193 static constexpr state_type WRITER_PENDING = 2; 194 static constexpr state_type READERS = ~(WRITER | WRITER_PENDING); 195 static constexpr state_type ONE_READER = 4; 196 static constexpr state_type BUSY = WRITER | READERS; 197 198 using context_type = std::uintptr_t; 199 static constexpr context_type WRITER_CONTEXT = 0; 200 static constexpr context_type READER_CONTEXT = 1; 201 friend scoped_lock; 202 //! State of lock 203 /** Bit 0 = writer is holding lock 204 Bit 1 = request by a writer to acquire lock (hint to readers to wait) 205 Bit 2..N = number of readers holding lock */ 206 std::atomic<state_type> m_state; 207 }; // class rw_mutex 208 209 } // namespace d1 210 } // namespace detail 211 212 inline namespace v1 { 213 using detail::d1::rw_mutex; 214 } // namespace v1 215 216 } // namespace tbb 217 218 #endif /* __TBB_PREVIEW_MUTEXES */ 219 220 #endif // __TBB_rw_mutex_H 221