1 /* 2 Copyright (c) 2021 Intel Corporation 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 #ifndef __TBB_rw_mutex_H 18 #define __TBB_rw_mutex_H 19 20 #include "detail/_namespace_injection.h" 21 #include "detail/_utils.h" 22 #include "detail/_waitable_atomic.h" 23 #include "detail/_scoped_lock.h" 24 #include "detail/_mutex_common.h" 25 #include "profiling.h" 26 27 namespace tbb { 28 namespace detail { 29 namespace d1 { 30 31 class rw_mutex { 32 public: 33 //! Constructors rw_mutex()34 rw_mutex() noexcept : m_state(0) { 35 create_itt_sync(this, "tbb::rw_mutex", ""); 36 } 37 38 //! Destructor ~rw_mutex()39 ~rw_mutex() { 40 __TBB_ASSERT(!m_state.load(std::memory_order_relaxed), "destruction of an acquired mutex"); 41 } 42 43 //! No Copy 44 rw_mutex(const rw_mutex&) = delete; 45 rw_mutex& operator=(const rw_mutex&) = delete; 46 47 using scoped_lock = rw_scoped_lock<rw_mutex>; 48 49 //! Mutex traits 50 static constexpr bool is_rw_mutex = true; 51 static constexpr bool is_recursive_mutex = false; 52 static constexpr bool is_fair_mutex = false; 53 54 //! Acquire lock lock()55 void lock() { 56 call_itt_notify(prepare, this); 57 while (!try_lock()) { 58 if (!(m_state.load(std::memory_order_relaxed) & WRITER_PENDING)) { // no pending writers 59 m_state |= WRITER_PENDING; 60 } 61 62 auto wakeup_condition = [&] { return !(m_state.load(std::memory_order_relaxed) & BUSY); }; 63 adaptive_wait_on_address(this, wakeup_condition, WRITER_CONTEXT); 64 } 65 66 call_itt_notify(acquired, this); 67 } 68 69 //! Try acquiring lock (non-blocking) 70 /** Return true if lock acquired; false otherwise. */ try_lock()71 bool try_lock() { 72 // for a writer: only possible to acquire if no active readers or writers 73 // Use relaxed memory fence is OK here because 74 // Acquire memory fence guaranteed by compare_exchange_strong() 75 state_type s = m_state.load(std::memory_order_relaxed); 76 if (!(s & BUSY)) { // no readers, no writers; mask is 1..1101 77 if (m_state.compare_exchange_strong(s, WRITER)) { 78 call_itt_notify(acquired, this); 79 return true; // successfully stored writer flag 80 } 81 } 82 return false; 83 } 84 85 //! Release lock unlock()86 void unlock() { 87 call_itt_notify(releasing, this); 88 state_type curr_state = (m_state &= READERS | WRITER_PENDING); // Returns current state 89 90 if (curr_state & WRITER_PENDING) { 91 r1::notify_by_address(this, WRITER_CONTEXT); 92 } else { 93 // It's possible that WRITER sleeps without WRITER_PENDING, 94 // because other thread might clear this bit at upgrade() 95 r1::notify_by_address_all(this); 96 } 97 } 98 99 //! Lock shared ownership mutex lock_shared()100 void lock_shared() { 101 call_itt_notify(prepare, this); 102 while (!try_lock_shared()) { 103 state_type has_writer = WRITER | WRITER_PENDING; 104 auto wakeup_condition = [&] { return !(m_state.load(std::memory_order_relaxed) & has_writer); }; 105 adaptive_wait_on_address(this, wakeup_condition, READER_CONTEXT); 106 } 107 __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state of a read lock: no readers"); 108 } 109 110 //! Try lock shared ownership mutex try_lock_shared()111 bool try_lock_shared() { 112 // for a reader: acquire if no active or waiting writers 113 // Use relaxed memory fence is OK here because 114 // Acquire memory fence guaranteed by fetch_add() 115 state_type has_writer = WRITER | WRITER_PENDING; 116 if (!(m_state.load(std::memory_order_relaxed) & has_writer)) { 117 if (m_state.fetch_add(ONE_READER) & has_writer) { 118 m_state -= ONE_READER; 119 r1::notify_by_address(this, WRITER_CONTEXT); 120 } else { 121 call_itt_notify(acquired, this); 122 return true; // successfully stored increased number of readers 123 } 124 } 125 return false; 126 } 127 128 //! Unlock shared ownership mutex unlock_shared()129 void unlock_shared() { 130 __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state of a read lock: no readers"); 131 call_itt_notify(releasing, this); 132 133 state_type curr_state = (m_state -= ONE_READER); // Returns current state 134 135 if (curr_state & (WRITER_PENDING)) { 136 r1::notify_by_address(this, WRITER_CONTEXT); 137 } else { 138 // It's possible that WRITER sleeps without WRITER_PENDING, 139 // because other thread might clear this bit at upgrade() 140 r1::notify_by_address_all(this); 141 } 142 } 143 144 private: 145 /** Internal non ISO C++ standard API **/ 146 //! This API is used through the scoped_lock class 147 148 //! Upgrade reader to become a writer. 149 /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ upgrade()150 bool upgrade() { 151 state_type s = m_state.load(std::memory_order_relaxed); 152 __TBB_ASSERT(s & READERS, "invalid state before upgrade: no readers "); 153 // Check and set writer-pending flag. 154 // Required conditions: either no pending writers, or we are the only reader 155 // (with multiple readers and pending writer, another upgrade could have been requested) 156 while ((s & READERS) == ONE_READER || !(s & WRITER_PENDING)) { 157 if (m_state.compare_exchange_strong(s, s | WRITER | WRITER_PENDING)) { 158 auto wakeup_condition = [&] { return (m_state.load(std::memory_order_relaxed) & READERS) == ONE_READER; }; 159 while ((m_state.load(std::memory_order_relaxed) & READERS) != ONE_READER) { 160 adaptive_wait_on_address(this, wakeup_condition, WRITER_CONTEXT); 161 } 162 163 __TBB_ASSERT((m_state.load(std::memory_order_relaxed) & (WRITER_PENDING|WRITER)) == (WRITER_PENDING | WRITER), 164 "invalid state when upgrading to writer"); 165 // Both new readers and writers are blocked at this time 166 m_state -= (ONE_READER + WRITER_PENDING); 167 return true; // successfully upgraded 168 } 169 } 170 // Slow reacquire 171 unlock_shared(); 172 lock(); 173 return false; 174 } 175 176 //! Downgrade writer to a reader downgrade()177 void downgrade() { 178 __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & WRITER, nullptr), 179 call_itt_notify(releasing, this); 180 m_state += (ONE_READER - WRITER); 181 182 if (!(m_state & WRITER_PENDING)) { 183 r1::notify_by_address(this, READER_CONTEXT); 184 } 185 186 __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state after downgrade: no readers"); 187 } 188 189 using state_type = std::intptr_t; 190 static constexpr state_type WRITER = 1; 191 static constexpr state_type WRITER_PENDING = 2; 192 static constexpr state_type READERS = ~(WRITER | WRITER_PENDING); 193 static constexpr state_type ONE_READER = 4; 194 static constexpr state_type BUSY = WRITER | READERS; 195 196 using context_type = std::uintptr_t; 197 static constexpr context_type WRITER_CONTEXT = 0; 198 static constexpr context_type READER_CONTEXT = 1; 199 friend scoped_lock; 200 //! State of lock 201 /** Bit 0 = writer is holding lock 202 Bit 1 = request by a writer to acquire lock (hint to readers to wait) 203 Bit 2..N = number of readers holding lock */ 204 std::atomic<state_type> m_state; 205 }; // class rw_mutex 206 207 } // namespace d1 208 } // namespace detail 209 210 inline namespace v1 { 211 using detail::d1::rw_mutex; 212 } // namespace v1 213 214 } // namespace tbb 215 216 #endif // __TBB_rw_mutex_H 217