Lines Matching refs:tricky_pointer

80 using tricky_pointer = tricky_atomic_pointer<queuing_rw_mutex::scoped_lock>;  typedef
147 static const tricky_pointer::word FLAG = 0x1;
181 predecessor = tricky_pointer(predecessor) & ~FLAG; in acquire()
183 tricky_pointer::store(predecessor->my_next, &s, std::memory_order_release); in acquire()
195 if( tricky_pointer(predecessor) & FLAG ) { in acquire()
198 predecessor = tricky_pointer(predecessor) & ~FLAG; in acquire()
213 tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed); in acquire()
214 … __TBB_ASSERT( !( tricky_pointer(predecessor) & FLAG ), "use of corrupted pointer!" ); in acquire()
216 tricky_pointer::store(predecessor->my_next, &s, std::memory_order_release); in acquire()
245tricky_pointer::load(s.my_next, std::memory_order_relaxed)->my_going.store(1U, std::memory_order_r… in acquire()
292 …d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquir… in release()
303 next = tricky_pointer::load(s.my_next, std::memory_order_acquire); in release()
319 …d1::queuing_rw_mutex::scoped_lock* tmp = tricky_pointer::exchange(next->my_prev, nullptr, std::mem… in release()
330 tricky_pointer::store(next->my_prev, nullptr, std::memory_order_release); in release()
342 …queuing_rw_mutex::scoped_lock *predecessor = tricky_pointer::fetch_add(s.my_prev, FLAG, std::memor… in release()
350 …tmp = tricky_pointer::compare_exchange_strong(s.my_prev, tricky_pointer(predecessor) | FLAG, prede… in release()
351 if( !(tricky_pointer(tmp) & FLAG) ) { in release()
352 …__TBB_ASSERT(tricky_pointer::load(s.my_prev, std::memory_order_relaxed) != (tricky_pointer(predece… in release()
362 tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed); in release()
365 tricky_pointer::store(predecessor->my_next, nullptr, std::memory_order_release); in release()
368 …if( !tricky_pointer::load(s.my_next, std::memory_order_acquire) && !s.my_mutex->q_tail.compare_exc… in release()
374 …if(d1::queuing_rw_mutex::scoped_lock *const l_next = tricky_pointer::load(s.my_next, std::memory_o… in release()
376 … tmp = tricky_pointer::exchange(l_next->my_prev, predecessor, std::memory_order_release); in release()
378 … __TBB_ASSERT(tricky_pointer::load(s.my_prev, std::memory_order_relaxed)==predecessor, nullptr); in release()
386 …d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquir… in release()
394 next = tricky_pointer::load(s.my_next, std::memory_order_acquire); in release()
401 tmp = tricky_pointer::exchange(next->my_prev, nullptr, std::memory_order_release); in release()
418 …d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquir… in downgrade_to_reader()
434 next = tricky_pointer::load(s.my_next, std::memory_order_acquire); in downgrade_to_reader()
474 …if( !s.my_mutex->q_tail.compare_exchange_strong(expected, tricky_pointer(me)|FLAG, std::memory_ord… in upgrade_to_writer()
477 next = tricky_pointer::fetch_add(s.my_next, FLAG, std::memory_order_acquire); in upgrade_to_writer()
486 tmp = tricky_pointer::exchange(next->my_prev, &s, std::memory_order_release); in upgrade_to_writer()
490 tmp = tricky_pointer(next)|FLAG; in upgrade_to_writer()
491 …for( atomic_backoff b; tricky_pointer::load(s.my_next, std::memory_order_relaxed)==tmp; b.pause() … in upgrade_to_writer()
493 if( tricky_pointer::load(s.my_next, std::memory_order_acquire)==tmp ) in upgrade_to_writer()
494 tricky_pointer::store(s.my_next, next, std::memory_order_relaxed); in upgrade_to_writer()
498 …__TBB_ASSERT(tricky_pointer::load(s.my_next, std::memory_order_relaxed) != (tricky_pointer(next)|F… in upgrade_to_writer()
502 …__TBB_ASSERT( (tricky_pointer(next)|FLAG) == tricky_pointer::load(s.my_next, std::memory_order_rel… in upgrade_to_writer()
503 tricky_pointer::store(s.my_next, next, std::memory_order_relaxed); in upgrade_to_writer()
523 expected = tricky_pointer(me)|FLAG; in upgrade_to_writer()
527 predecessor = tricky_pointer::fetch_add(s.my_prev, FLAG, std::memory_order_acquire); in upgrade_to_writer()
541 …tmp = tricky_pointer::compare_exchange_strong(s.my_prev, tricky_pointer(predecessor)|FLAG, predece… in upgrade_to_writer()
542 if( tricky_pointer(tmp) & FLAG ) { in upgrade_to_writer()
543 tricky_pointer::spin_wait_while_eq(s.my_prev, predecessor); in upgrade_to_writer()
544 predecessor = tricky_pointer::load(s.my_prev, std::memory_order_relaxed); in upgrade_to_writer()
547 tricky_pointer::spin_wait_while_eq(s.my_prev, tricky_pointer(predecessor)|FLAG); in upgrade_to_writer()
551 tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed); in upgrade_to_writer()
553 tricky_pointer::spin_wait_while_eq(s.my_prev, predecessor); in upgrade_to_writer()
554 predecessor = tricky_pointer::load(s.my_prev, std::memory_order_relaxed); in upgrade_to_writer()
559 tricky_pointer::store(s.my_prev, nullptr, std::memory_order_relaxed); in upgrade_to_writer()