37     static void release(d1::rtm_rw_mutex::scoped_lock& s) {  in release()
38         switch(s.m_transaction_state) {  in release()
43             s.m_mutex = nullptr;  in release()
46 …__TBB_ASSERT(!s.m_mutex->write_flag.load(std::memory_order_relaxed), "write_flag set but read lock…  in release()
47             s.m_mutex->unlock_shared();  in release()
48             s.m_mutex = nullptr;  in release()
51 …__TBB_ASSERT(s.m_mutex->write_flag.load(std::memory_order_relaxed), "write_flag unset but write lo…  in release()
52             s.m_mutex->write_flag.store(false, std::memory_order_relaxed);  in release()
53             s.m_mutex->unlock();  in release()
54             s.m_mutex = nullptr;  in release()
62         s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex;  in release()
66 …static void acquire_writer(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s, bool only_specul…  in acquire_writer()
67 …__TBB_ASSERT(s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex, "scoped_lock a…  in acquire_writer()
84                     s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer;  in acquire_writer()
87                     s.m_mutex = &m;  in acquire_writer()
95 …s.m_mutex = &m;                                                          // should apply a real tr…  in acquire_writer()
96 …s.m_mutex->lock();                                                       // kill transactional wri…  in acquire_writer()
99         s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_writer;  in acquire_writer()
106 …static void acquire_reader(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s, bool only_specul…  in acquire_reader()
107 …__TBB_ASSERT(s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex, "scoped_lock a…  in acquire_reader()
124                     s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader;  in acquire_reader()
127                     s.m_mutex = &m;  in acquire_reader()
144         s.m_mutex = &m;  in acquire_reader()
145         s.m_mutex->lock_shared();  in acquire_reader()
146         s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_reader;  in acquire_reader()
151     static bool upgrade(d1::rtm_rw_mutex::scoped_lock& s) {  in upgrade()
152         switch(s.m_transaction_state) {  in upgrade()
154             s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_writer;  in upgrade()
155             bool no_release = s.m_mutex->upgrade();  in upgrade()
156 …__TBB_ASSERT(!s.m_mutex->write_flag.load(std::memory_order_relaxed), "After upgrade, write_flag al…  in upgrade()
157             s.m_mutex->write_flag.store(true, std::memory_order_relaxed);  in upgrade()
161             d1::rtm_rw_mutex& m = *s.m_mutex;  in upgrade()
164                 release(s);  in upgrade()
165                 acquire_writer(m, s, false);  in upgrade()
169                 s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer;  in upgrade()
180     static bool downgrade(d1::rtm_rw_mutex::scoped_lock& s) {  in downgrade()
181         switch (s.m_transaction_state) {  in downgrade()
183             s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_reader;  in downgrade()
184 …__TBB_ASSERT(s.m_mutex->write_flag.load(std::memory_order_relaxed), "Before downgrade write_flag n…  in downgrade()
185             s.m_mutex->write_flag.store(false, std::memory_order_relaxed);  in downgrade()
186             s.m_mutex->downgrade();  in downgrade()
189             s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader;  in downgrade()
202     static bool try_acquire_writer(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s) {  in try_acquire_writer()
203         acquire_writer(m, s, /*only_speculate=*/true);  in try_acquire_writer()
204         if (s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer) {  in try_acquire_writer()
207 …      __TBB_ASSERT(s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex, nullptr);  in try_acquire_writer()
210             s.m_mutex = &m;  in try_acquire_writer()
214             s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_writer;  in try_acquire_writer()
221     static bool try_acquire_reader(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s) {  in try_acquire_reader()
223         acquire_reader(m, s, /*only_speculate=*/true);  in try_acquire_reader()
224         if (s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader) {  in try_acquire_reader()
227 …      __TBB_ASSERT(s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex, nullptr);  in try_acquire_reader()
230             s.m_mutex = &m;  in try_acquire_reader()
231             s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_reader;  in try_acquire_reader()
238 void __TBB_EXPORTED_FUNC acquire_writer(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s, bool…  in acquire_writer()  argument
239     rtm_rw_mutex_impl::acquire_writer(m, s, only_speculate);  in acquire_writer()
243 void __TBB_EXPORTED_FUNC acquire_reader(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s, bool…  in acquire_reader()  argument
244     rtm_rw_mutex_impl::acquire_reader(m, s, only_speculate);  in acquire_reader()
247 bool __TBB_EXPORTED_FUNC upgrade(d1::rtm_rw_mutex::scoped_lock& s) {  in upgrade()  argument
248     return rtm_rw_mutex_impl::upgrade(s);  in upgrade()
251 bool __TBB_EXPORTED_FUNC downgrade(d1::rtm_rw_mutex::scoped_lock& s) {  in downgrade()  argument
252     return rtm_rw_mutex_impl::downgrade(s);  in downgrade()
255 bool __TBB_EXPORTED_FUNC try_acquire_writer(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s) {  in try_acquire_writer()  argument
256     return rtm_rw_mutex_impl::try_acquire_writer(m, s);  in try_acquire_writer()
259 bool __TBB_EXPORTED_FUNC try_acquire_reader(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s) {  in try_acquire_reader()  argument
260     return rtm_rw_mutex_impl::try_acquire_reader(m, s);  in try_acquire_reader()
263 void __TBB_EXPORTED_FUNC release(d1::rtm_rw_mutex::scoped_lock& s) {  in release()  argument
264     rtm_rw_mutex_impl::release(s);  in release()