Lines Matching refs:s
108 static bool try_acquire_internal_lock(d1::queuing_rw_mutex::scoped_lock& s) in try_acquire_internal_lock()
111 return s.my_internal_lock.compare_exchange_strong(expected, ACQUIRED); in try_acquire_internal_lock()
115 static void acquire_internal_lock(d1::queuing_rw_mutex::scoped_lock& s) in acquire_internal_lock()
119 while( !try_acquire_internal_lock(s) ) { in acquire_internal_lock()
125 static void release_internal_lock(d1::queuing_rw_mutex::scoped_lock& s) in release_internal_lock()
127 s.my_internal_lock.store(RELEASED, std::memory_order_release); in release_internal_lock()
131 static void wait_for_release_of_internal_lock(d1::queuing_rw_mutex::scoped_lock& s) in wait_for_release_of_internal_lock()
133 spin_wait_until_eq(s.my_internal_lock, RELEASED); in wait_for_release_of_internal_lock()
137 …static void unblock_or_wait_on_internal_lock(d1::queuing_rw_mutex::scoped_lock& s, uintptr_t flag … in unblock_or_wait_on_internal_lock()
139 wait_for_release_of_internal_lock(s); in unblock_or_wait_on_internal_lock()
142 release_internal_lock(s); in unblock_or_wait_on_internal_lock()
158 static void acquire(d1::queuing_rw_mutex& m, d1::queuing_rw_mutex::scoped_lock& s, bool write) in acquire()
160 __TBB_ASSERT( !s.my_mutex, "scoped_lock is already holding a mutex"); in acquire()
164 s.my_mutex = &m; in acquire()
165 s.my_prev.store(0U, std::memory_order_relaxed); in acquire()
166 s.my_next.store(0U, std::memory_order_relaxed); in acquire()
167 s.my_going.store(0U, std::memory_order_relaxed); in acquire()
168 …s.my_state.store(d1::queuing_rw_mutex::scoped_lock::state_t(write ? STATE_WRITER : STATE_READER), … in acquire()
169 s.my_internal_lock.store(RELEASED, std::memory_order_relaxed); in acquire()
175 … queuing_rw_mutex::scoped_lock* predecessor = m.q_tail.exchange(&s, std::memory_order_acq_rel); in acquire()
180 ITT_NOTIFY(sync_prepare, s.my_mutex); in acquire()
183 tricky_pointer::store(predecessor->my_next, &s, std::memory_order_release); in acquire()
185 spin_wait_until_eq(s.my_going, 1U, std::memory_order_acquire); in acquire()
194 … __TBB_ASSERT( !s.my_prev.load(std::memory_order_relaxed), "the predecessor is already set" ); in acquire()
213 tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed); in acquire()
216 tricky_pointer::store(predecessor->my_next, &s, std::memory_order_release); in acquire()
220 ITT_NOTIFY(sync_prepare, s.my_mutex); in acquire()
223 spin_wait_until_eq(s.my_going, 1U, std::memory_order_acquire); in acquire()
233 …s.my_state.compare_exchange_strong(old_state, STATE_ACTIVEREADER, std::memory_order_release, std::… in acquire()
237 ITT_NOTIFY(sync_prepare, s.my_mutex); in acquire()
240 …__TBB_ASSERT( s.my_state.load(std::memory_order_relaxed)==STATE_READER_UNBLOCKNEXT, "unexpected st… in acquire()
241 spin_wait_while_eq(s.my_next, 0U, std::memory_order_acquire); in acquire()
244 s.my_state.store(STATE_ACTIVEREADER, std::memory_order_relaxed); in acquire()
245 …tricky_pointer::load(s.my_next, std::memory_order_relaxed)->my_going.store(1U, std::memory_order_r… in acquire()
247 …__TBB_ASSERT(s.my_state.load(std::memory_order_relaxed) == STATE_ACTIVEREADER, "unlocked reader is… in acquire()
250 ITT_NOTIFY(sync_acquired, s.my_mutex); in acquire()
254 … static bool try_acquire(d1::queuing_rw_mutex& m, d1::queuing_rw_mutex::scoped_lock& s, bool write) in try_acquire()
256 __TBB_ASSERT( !s.my_mutex, "scoped_lock is already holding a mutex"); in try_acquire()
263 s.my_prev.store(0U, std::memory_order_relaxed); in try_acquire()
264 s.my_next.store(0U, std::memory_order_relaxed); in try_acquire()
265 s.my_going.store(0U, std::memory_order_relaxed); // TODO: remove dead assignment? in try_acquire()
266 …s.my_state.store(d1::queuing_rw_mutex::scoped_lock::state_t(write ? STATE_WRITER : STATE_ACTIVEREA… in try_acquire()
267 s.my_internal_lock.store(RELEASED, std::memory_order_relaxed); in try_acquire()
273 if (!m.q_tail.compare_exchange_strong(expected, &s, std::memory_order_acq_rel)) in try_acquire()
275 s.my_mutex = &m; in try_acquire()
276 ITT_NOTIFY(sync_acquired, s.my_mutex); in try_acquire()
281 static void release(d1::queuing_rw_mutex::scoped_lock& s) { in release()
282 __TBB_ASSERT(s.my_mutex!=nullptr, "no lock acquired"); in release()
284 ITT_NOTIFY(sync_releasing, s.my_mutex); in release()
286 if( s.my_state.load(std::memory_order_relaxed) == STATE_WRITER ) { // Acquired for write in release()
292 …d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquir… in release()
294 d1::queuing_rw_mutex::scoped_lock* expected = &s; in release()
296 if( s.my_mutex->q_tail.compare_exchange_strong(expected, nullptr, in release()
302 spin_wait_while_eq(s.my_next, 0U, std::memory_order_relaxed); in release()
303 next = tricky_pointer::load(s.my_next, std::memory_order_acquire); in release()
316 acquire_internal_lock(s); in release()
324 unblock_or_wait_on_internal_lock(s, get_flag(tmp)); in release()
342 …queuing_rw_mutex::scoped_lock *predecessor = tricky_pointer::fetch_add(s.my_prev, FLAG, std::memor… in release()
350 …tmp = tricky_pointer::compare_exchange_strong(s.my_prev, tricky_pointer(predecessor) | FLAG, prede… in release()
352 …__TBB_ASSERT(tricky_pointer::load(s.my_prev, std::memory_order_relaxed) != (tricky_pointer(predece… in release()
362 tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed); in release()
363 acquire_internal_lock(s); in release()
367 d1::queuing_rw_mutex::scoped_lock* expected = &s; in release()
368 …if( !tricky_pointer::load(s.my_next, std::memory_order_acquire) && !s.my_mutex->q_tail.compare_exc… in release()
369 spin_wait_while_eq( s.my_next, 0U, std::memory_order_acquire ); in release()
371 … __TBB_ASSERT( !(s.my_next.load(std::memory_order_relaxed) & FLAG), "use of corrupted pointer" ); in release()
374 …if(d1::queuing_rw_mutex::scoped_lock *const l_next = tricky_pointer::load(s.my_next, std::memory_o… in release()
378 … __TBB_ASSERT(tricky_pointer::load(s.my_prev, std::memory_order_relaxed)==predecessor, nullptr); in release()
379 … predecessor->my_next.store(s.my_next.load(std::memory_order_relaxed), std::memory_order_release); in release()
385 acquire_internal_lock(s); // "exclusiveLock(&I->EL)" in release()
386 …d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquir… in release()
388 d1::queuing_rw_mutex::scoped_lock* expected = &s; in release()
390 if( !s.my_mutex->q_tail.compare_exchange_strong(expected, nullptr, in release()
393 spin_wait_while_eq( s.my_next, 0U, std::memory_order_relaxed ); in release()
394 next = tricky_pointer::load(s.my_next, std::memory_order_acquire); in release()
405 unblock_or_wait_on_internal_lock(s, get_flag(tmp)); in release()
409 spin_wait_while_eq( s.my_going, 2U, std::memory_order_relaxed ); in release()
411 s.initialize(); in release()
414 static bool downgrade_to_reader(d1::queuing_rw_mutex::scoped_lock& s) { in downgrade_to_reader()
415 …if ( s.my_state.load(std::memory_order_relaxed) == STATE_ACTIVEREADER ) return true; // Already a … in downgrade_to_reader()
417 ITT_NOTIFY(sync_releasing, s.my_mutex); in downgrade_to_reader()
418 …d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquir… in downgrade_to_reader()
420 s.my_state.store(STATE_READER, std::memory_order_seq_cst); in downgrade_to_reader()
422 if( &s == s.my_mutex->q_tail.load(std::memory_order_seq_cst) ) { in downgrade_to_reader()
428 …s.my_state.compare_exchange_strong(old_state, STATE_ACTIVEREADER, std::memory_order_release, std::… in downgrade_to_reader()
433 spin_wait_while_eq(s.my_next, 0U, std::memory_order_relaxed); in downgrade_to_reader()
434 next = tricky_pointer::load(s.my_next, std::memory_order_acquire); in downgrade_to_reader()
452 s.my_state.store(STATE_ACTIVEREADER, std::memory_order_release); in downgrade_to_reader()
456 static bool upgrade_to_writer(d1::queuing_rw_mutex::scoped_lock& s) { in upgrade_to_writer()
457 if (s.my_state.load(std::memory_order_relaxed) == STATE_WRITER) { in upgrade_to_writer()
462 …__TBB_ASSERT(s.my_state.load(std::memory_order_relaxed) == STATE_ACTIVEREADER, "only active reader… in upgrade_to_writer()
465 queuing_rw_mutex::scoped_lock* me = &s; in upgrade_to_writer()
467 ITT_NOTIFY(sync_releasing, s.my_mutex); in upgrade_to_writer()
469 s.my_state.store(STATE_UPGRADE_REQUESTED, std::memory_order_release); in upgrade_to_writer()
471 … __TBB_ASSERT( !(s.my_next.load(std::memory_order_relaxed) & FLAG), "use of corrupted pointer!" ); in upgrade_to_writer()
472 acquire_internal_lock(s); in upgrade_to_writer()
473 d1::queuing_rw_mutex::scoped_lock* expected = &s; in upgrade_to_writer()
474 …if( !s.my_mutex->q_tail.compare_exchange_strong(expected, tricky_pointer(me)|FLAG, std::memory_ord… in upgrade_to_writer()
475 spin_wait_while_eq( s.my_next, 0U, std::memory_order_relaxed ); in upgrade_to_writer()
477 next = tricky_pointer::fetch_add(s.my_next, FLAG, std::memory_order_acquire); in upgrade_to_writer()
486 tmp = tricky_pointer::exchange(next->my_prev, &s, std::memory_order_release); in upgrade_to_writer()
487 unblock_or_wait_on_internal_lock(s, get_flag(tmp)); in upgrade_to_writer()
491 …for( atomic_backoff b; tricky_pointer::load(s.my_next, std::memory_order_relaxed)==tmp; b.pause() … in upgrade_to_writer()
492 if( s.my_state.load(std::memory_order_acquire) & STATE_COMBINED_UPGRADING ) { in upgrade_to_writer()
493 if( tricky_pointer::load(s.my_next, std::memory_order_acquire)==tmp ) in upgrade_to_writer()
494 tricky_pointer::store(s.my_next, next, std::memory_order_relaxed); in upgrade_to_writer()
498 …__TBB_ASSERT(tricky_pointer::load(s.my_next, std::memory_order_relaxed) != (tricky_pointer(next)|F… in upgrade_to_writer()
502 …__TBB_ASSERT( (tricky_pointer(next)|FLAG) == tricky_pointer::load(s.my_next, std::memory_order_rel… in upgrade_to_writer()
503 tricky_pointer::store(s.my_next, next, std::memory_order_relaxed); in upgrade_to_writer()
507 release_internal_lock(s); in upgrade_to_writer()
515 …s.my_state.compare_exchange_strong(old_state, STATE_UPGRADE_WAITING, std::memory_order_release, st… in upgrade_to_writer()
518 …__TBB_ASSERT( !( s.my_next.load(std::memory_order_relaxed) & FLAG ), "use of corrupted pointer!" ); in upgrade_to_writer()
519 … __TBB_ASSERT( s.my_state & STATE_COMBINED_UPGRADING, "wrong state at upgrade waiting_retry" ); in upgrade_to_writer()
520 __TBB_ASSERT( me==&s, nullptr ); in upgrade_to_writer()
521 ITT_NOTIFY(sync_prepare, s.my_mutex); in upgrade_to_writer()
524 s.my_mutex->q_tail.compare_exchange_strong(expected, &s, std::memory_order_release); in upgrade_to_writer()
527 predecessor = tricky_pointer::fetch_add(s.my_prev, FLAG, std::memory_order_acquire); in upgrade_to_writer()
541 …tmp = tricky_pointer::compare_exchange_strong(s.my_prev, tricky_pointer(predecessor)|FLAG, predece… in upgrade_to_writer()
543 tricky_pointer::spin_wait_while_eq(s.my_prev, predecessor); in upgrade_to_writer()
544 predecessor = tricky_pointer::load(s.my_prev, std::memory_order_relaxed); in upgrade_to_writer()
547 tricky_pointer::spin_wait_while_eq(s.my_prev, tricky_pointer(predecessor)|FLAG); in upgrade_to_writer()
551 tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed); in upgrade_to_writer()
553 tricky_pointer::spin_wait_while_eq(s.my_prev, predecessor); in upgrade_to_writer()
554 predecessor = tricky_pointer::load(s.my_prev, std::memory_order_relaxed); in upgrade_to_writer()
559 tricky_pointer::store(s.my_prev, nullptr, std::memory_order_relaxed); in upgrade_to_writer()
561 __TBB_ASSERT( !predecessor && !s.my_prev, nullptr ); in upgrade_to_writer()
565 wait_for_release_of_internal_lock(s); in upgrade_to_writer()
567 spin_wait_while_eq( s.my_going, 2U ); in upgrade_to_writer()
569 bool result = ( s.my_state != STATE_UPGRADE_LOSER ); in upgrade_to_writer()
570 s.my_state.store(STATE_WRITER, std::memory_order_relaxed); in upgrade_to_writer()
571 s.my_going.store(1U, std::memory_order_relaxed); in upgrade_to_writer()
573 ITT_NOTIFY(sync_acquired, s.my_mutex); in upgrade_to_writer()
587 void __TBB_EXPORTED_FUNC acquire(d1::queuing_rw_mutex& m, d1::queuing_rw_mutex::scoped_lock& s, boo… in acquire() argument
588 queuing_rw_mutex_impl::acquire(m, s, write); in acquire()
591 bool __TBB_EXPORTED_FUNC try_acquire(d1::queuing_rw_mutex& m, d1::queuing_rw_mutex::scoped_lock& s,… in try_acquire() argument
592 return queuing_rw_mutex_impl::try_acquire(m, s, write); in try_acquire()
595 void __TBB_EXPORTED_FUNC release(d1::queuing_rw_mutex::scoped_lock& s) { in release() argument
596 queuing_rw_mutex_impl::release(s); in release()
599 bool __TBB_EXPORTED_FUNC upgrade_to_writer(d1::queuing_rw_mutex::scoped_lock& s) { in upgrade_to_writer() argument
600 return queuing_rw_mutex_impl::upgrade_to_writer(s); in upgrade_to_writer()
603 bool __TBB_EXPORTED_FUNC is_writer(const d1::queuing_rw_mutex::scoped_lock& s) { in is_writer() argument
604 return queuing_rw_mutex_impl::is_writer(s); in is_writer()
607 bool __TBB_EXPORTED_FUNC downgrade_to_reader(d1::queuing_rw_mutex::scoped_lock& s) { in downgrade_to_reader() argument
608 return queuing_rw_mutex_impl::downgrade_to_reader(s); in downgrade_to_reader()