Lines Matching refs:std
42 … static T* fetch_add( std::atomic<word>& location, word addend, std::memory_order memory_order ) { in fetch_add()
46 static T* exchange( std::atomic<word>& location, T* value, std::memory_order memory_order ) { in exchange()
50 …static T* compare_exchange_strong( std::atomic<word>& obj, const T* expected, const T* desired, st… in compare_exchange_strong()
56 … static void store( std::atomic<word>& location, const T* value, std::memory_order memory_order ) { in store()
60 static T* load( std::atomic<word>& location, std::memory_order memory_order ) { in load()
64 static void spin_wait_while_eq(const std::atomic<word>& location, const T* value) { in spin_wait_while_eq()
127 s.my_internal_lock.store(RELEASED, std::memory_order_release); in release_internal_lock()
165 s.my_prev.store(0U, std::memory_order_relaxed); in acquire()
166 s.my_next.store(0U, std::memory_order_relaxed); in acquire()
167 s.my_going.store(0U, std::memory_order_relaxed); in acquire()
168 …euing_rw_mutex::scoped_lock::state_t(write ? STATE_WRITER : STATE_READER), std::memory_order_relax… in acquire()
169 s.my_internal_lock.store(RELEASED, std::memory_order_relaxed); in acquire()
175 … queuing_rw_mutex::scoped_lock* predecessor = m.q_tail.exchange(&s, std::memory_order_acq_rel); in acquire()
183 tricky_pointer::store(predecessor->my_next, &s, std::memory_order_release); in acquire()
185 spin_wait_until_eq(s.my_going, 1U, std::memory_order_acquire); in acquire()
194 … __TBB_ASSERT( !s.my_prev.load(std::memory_order_relaxed), "the predecessor is already set" ); in acquire()
202 pred_state = predecessor->my_state.load(std::memory_order_relaxed); in acquire()
205 …or->my_state.compare_exchange_strong(pred_state, STATE_READER_UNBLOCKNEXT, std::memory_order_relax… in acquire()
210 (void)predecessor->my_state.load(std::memory_order_acquire); in acquire()
213 tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed); in acquire()
215 …__TBB_ASSERT( !predecessor->my_next.load(std::memory_order_relaxed), "the predecessor has another … in acquire()
216 tricky_pointer::store(predecessor->my_next, &s, std::memory_order_release); in acquire()
223 spin_wait_until_eq(s.my_going, 1U, std::memory_order_acquire); in acquire()
233 …state.compare_exchange_strong(old_state, STATE_ACTIVEREADER, std::memory_order_release, std::memor… in acquire()
240 …__TBB_ASSERT( s.my_state.load(std::memory_order_relaxed)==STATE_READER_UNBLOCKNEXT, "unexpected st… in acquire()
241 spin_wait_while_eq(s.my_next, 0U, std::memory_order_acquire); in acquire()
244 s.my_state.store(STATE_ACTIVEREADER, std::memory_order_relaxed); in acquire()
245 …tricky_pointer::load(s.my_next, std::memory_order_relaxed)->my_going.store(1U, std::memory_order_r… in acquire()
247 …__TBB_ASSERT(s.my_state.load(std::memory_order_relaxed) == STATE_ACTIVEREADER, "unlocked reader is… in acquire()
258 if( m.q_tail.load(std::memory_order_relaxed) ) in try_acquire()
263 s.my_prev.store(0U, std::memory_order_relaxed); in try_acquire()
264 s.my_next.store(0U, std::memory_order_relaxed); in try_acquire()
265 s.my_going.store(0U, std::memory_order_relaxed); // TODO: remove dead assignment? in try_acquire()
266 …rw_mutex::scoped_lock::state_t(write ? STATE_WRITER : STATE_ACTIVEREADER), std::memory_order_relax… in try_acquire()
267 s.my_internal_lock.store(RELEASED, std::memory_order_relaxed); in try_acquire()
273 if (!m.q_tail.compare_exchange_strong(expected, &s, std::memory_order_acq_rel)) in try_acquire()
286 if( s.my_state.load(std::memory_order_relaxed) == STATE_WRITER ) { // Acquired for write in release()
292 …d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquir… in release()
297 std::memory_order_release, std::memory_order_relaxed) ) in release()
302 spin_wait_while_eq(s.my_next, 0U, std::memory_order_relaxed); in release()
303 next = tricky_pointer::load(s.my_next, std::memory_order_acquire); in release()
305 …next->my_going.store(2U, std::memory_order_relaxed); // protect next queue node from being destroy… in release()
314 if( next->my_state.load(std::memory_order_acquire)==STATE_UPGRADE_WAITING ) { in release()
319 …mutex::scoped_lock* tmp = tricky_pointer::exchange(next->my_prev, nullptr, std::memory_order_relea… in release()
321 next->my_state.store(STATE_UPGRADE_LOSER, std::memory_order_release); in release()
323 next->my_going.store(1U, std::memory_order_release); in release()
327 …__TBB_ASSERT( next->my_state.load(std::memory_order_relaxed) & (STATE_COMBINED_WAITINGREADER | STA… in release()
328 …__TBB_ASSERT( !( next->my_prev.load(std::memory_order_relaxed) & FLAG ), "use of corrupted pointer… in release()
330 tricky_pointer::store(next->my_prev, nullptr, std::memory_order_release); in release()
332 next->my_going.store(1U, std::memory_order_release); in release()
342 …tex::scoped_lock *predecessor = tricky_pointer::fetch_add(s.my_prev, FLAG, std::memory_order_acqui… in release()
350 …xchange_strong(s.my_prev, tricky_pointer(predecessor) | FLAG, predecessor, std::memory_order_acqui… in release()
352 …__TBB_ASSERT(tricky_pointer::load(s.my_prev, std::memory_order_relaxed) != (tricky_pointer(predece… in release()
361 …__TBB_ASSERT(predecessor && predecessor->my_internal_lock.load(std::memory_order_relaxed)==ACQUIRE… in release()
362 tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed); in release()
365 tricky_pointer::store(predecessor->my_next, nullptr, std::memory_order_release); in release()
368 …_pointer::load(s.my_next, std::memory_order_acquire) && !s.my_mutex->q_tail.compare_exchange_stron… in release()
369 spin_wait_while_eq( s.my_next, 0U, std::memory_order_acquire ); in release()
371 … __TBB_ASSERT( !(s.my_next.load(std::memory_order_relaxed) & FLAG), "use of corrupted pointer" ); in release()
374 …uing_rw_mutex::scoped_lock *const l_next = tricky_pointer::load(s.my_next, std::memory_order_relax… in release()
376 … tmp = tricky_pointer::exchange(l_next->my_prev, predecessor, std::memory_order_release); in release()
378 … __TBB_ASSERT(tricky_pointer::load(s.my_prev, std::memory_order_relaxed)==predecessor, nullptr); in release()
379 … predecessor->my_next.store(s.my_next.load(std::memory_order_relaxed), std::memory_order_release); in release()
386 …d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquir… in release()
391 std::memory_order_release, std::memory_order_relaxed) ) in release()
393 spin_wait_while_eq( s.my_next, 0U, std::memory_order_relaxed ); in release()
394 next = tricky_pointer::load(s.my_next, std::memory_order_acquire); in release()
399 next->my_going.store(2U, std::memory_order_relaxed); in release()
401 tmp = tricky_pointer::exchange(next->my_prev, nullptr, std::memory_order_release); in release()
402 next->my_going.store(1U, std::memory_order_release); in release()
409 spin_wait_while_eq( s.my_going, 2U, std::memory_order_relaxed ); in release()
415 …if ( s.my_state.load(std::memory_order_relaxed) == STATE_ACTIVEREADER ) return true; // Already a … in downgrade_to_reader()
418 …d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquir… in downgrade_to_reader()
420 s.my_state.store(STATE_READER, std::memory_order_seq_cst); in downgrade_to_reader()
422 if( &s == s.my_mutex->q_tail.load(std::memory_order_seq_cst) ) { in downgrade_to_reader()
428 …state.compare_exchange_strong(old_state, STATE_ACTIVEREADER, std::memory_order_release, std::memor… in downgrade_to_reader()
433 spin_wait_while_eq(s.my_next, 0U, std::memory_order_relaxed); in downgrade_to_reader()
434 next = tricky_pointer::load(s.my_next, std::memory_order_acquire); in downgrade_to_reader()
438 if( next->my_state.load(std::memory_order_relaxed) & STATE_COMBINED_WAITINGREADER ) in downgrade_to_reader()
439 next->my_going.store(1U, std::memory_order_release); in downgrade_to_reader()
448 else if( next->my_state.load(std::memory_order_acquire)==STATE_UPGRADE_WAITING ) in downgrade_to_reader()
451 next->my_state.store(STATE_UPGRADE_LOSER, std::memory_order_release); in downgrade_to_reader()
452 s.my_state.store(STATE_ACTIVEREADER, std::memory_order_release); in downgrade_to_reader()
457 if (s.my_state.load(std::memory_order_relaxed) == STATE_WRITER) { in upgrade_to_writer()
462 …__TBB_ASSERT(s.my_state.load(std::memory_order_relaxed) == STATE_ACTIVEREADER, "only active reader… in upgrade_to_writer()
469 s.my_state.store(STATE_UPGRADE_REQUESTED, std::memory_order_release); in upgrade_to_writer()
471 … __TBB_ASSERT( !(s.my_next.load(std::memory_order_relaxed) & FLAG), "use of corrupted pointer!" ); in upgrade_to_writer()
474 …if( !s.my_mutex->q_tail.compare_exchange_strong(expected, tricky_pointer(me)|FLAG, std::memory_ord… in upgrade_to_writer()
475 spin_wait_while_eq( s.my_next, 0U, std::memory_order_relaxed ); in upgrade_to_writer()
477 next = tricky_pointer::fetch_add(s.my_next, FLAG, std::memory_order_acquire); in upgrade_to_writer()
481 unsigned short n_state = next->my_state.load(std::memory_order_acquire); in upgrade_to_writer()
484 next->my_going.store(1U, std::memory_order_release); in upgrade_to_writer()
486 tmp = tricky_pointer::exchange(next->my_prev, &s, std::memory_order_release); in upgrade_to_writer()
491 …for( atomic_backoff b; tricky_pointer::load(s.my_next, std::memory_order_relaxed)==tmp; b.pause() … in upgrade_to_writer()
492 if( s.my_state.load(std::memory_order_acquire) & STATE_COMBINED_UPGRADING ) { in upgrade_to_writer()
493 if( tricky_pointer::load(s.my_next, std::memory_order_acquire)==tmp ) in upgrade_to_writer()
494 tricky_pointer::store(s.my_next, next, std::memory_order_relaxed); in upgrade_to_writer()
498 …__TBB_ASSERT(tricky_pointer::load(s.my_next, std::memory_order_relaxed) != (tricky_pointer(next)|F… in upgrade_to_writer()
502 …__TBB_ASSERT( (tricky_pointer(next)|FLAG) == tricky_pointer::load(s.my_next, std::memory_order_rel… in upgrade_to_writer()
503 tricky_pointer::store(s.my_next, next, std::memory_order_relaxed); in upgrade_to_writer()
515 …te.compare_exchange_strong(old_state, STATE_UPGRADE_WAITING, std::memory_order_release, std::memor… in upgrade_to_writer()
518 …__TBB_ASSERT( !( s.my_next.load(std::memory_order_relaxed) & FLAG ), "use of corrupted pointer!" ); in upgrade_to_writer()
524 s.my_mutex->q_tail.compare_exchange_strong(expected, &s, std::memory_order_release); in upgrade_to_writer()
527 predecessor = tricky_pointer::fetch_add(s.my_prev, FLAG, std::memory_order_acquire); in upgrade_to_writer()
536 …predecessor->my_state.compare_exchange_strong(old_state, STATE_UPGRADE_WAITING, std::memory_order_… in upgrade_to_writer()
537 std::memory_order_relaxed); in upgrade_to_writer()
541 …_exchange_strong(s.my_prev, tricky_pointer(predecessor)|FLAG, predecessor, std::memory_order_acqui… in upgrade_to_writer()
544 predecessor = tricky_pointer::load(s.my_prev, std::memory_order_relaxed); in upgrade_to_writer()
551 tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed); in upgrade_to_writer()
554 predecessor = tricky_pointer::load(s.my_prev, std::memory_order_relaxed); in upgrade_to_writer()
559 tricky_pointer::store(s.my_prev, nullptr, std::memory_order_relaxed); in upgrade_to_writer()
570 s.my_state.store(STATE_WRITER, std::memory_order_relaxed); in upgrade_to_writer()
571 s.my_going.store(1U, std::memory_order_relaxed); in upgrade_to_writer()
578 return m.my_state.load(std::memory_order_relaxed) == STATE_WRITER; in is_writer()