175 …     queuing_rw_mutex::scoped_lock* predecessor = m.q_tail.exchange(&s, std::memory_order_acq_rel);  in acquire()  local
179             if( predecessor ) {  in acquire()
181                 predecessor = tricky_pointer(predecessor) & ~FLAG;  in acquire()
182                 __TBB_ASSERT( !predecessor->my_next, "the predecessor has another successor!");  in acquire()
183                 tricky_pointer::store(predecessor->my_next, &s, std::memory_order_release);  in acquire()
192             if( predecessor ) {  in acquire()
195                 if( tricky_pointer(predecessor) & FLAG ) {  in acquire()
198                     predecessor = tricky_pointer(predecessor) & ~FLAG;  in acquire()
202                     pred_state = predecessor->my_state.load(std::memory_order_relaxed);  in acquire()
205 …predecessor->my_state.compare_exchange_strong(pred_state, STATE_READER_UNBLOCKNEXT, std::memory_or…  in acquire()
210                         (void)predecessor->my_state.load(std::memory_order_acquire);  in acquire()
213                 tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed);  in acquire()
214 …              __TBB_ASSERT( !( tricky_pointer(predecessor) & FLAG ), "use of corrupted pointer!" );  in acquire()
215 …__TBB_ASSERT( !predecessor->my_next.load(std::memory_order_relaxed), "the predecessor has another …  in acquire()
216                 tricky_pointer::store(predecessor->my_next, &s, std::memory_order_release);  in acquire()
342 …queuing_rw_mutex::scoped_lock *predecessor = tricky_pointer::fetch_add(s.my_prev, FLAG, std::memor…  in release()
344             if( predecessor ) {  in release()
345                 if( !(try_acquire_internal_lock(*predecessor)) )  in release()
350 …_pointer::compare_exchange_strong(s.my_prev, tricky_pointer(predecessor) | FLAG, predecessor, std:…  in release()
352 …pointer::load(s.my_prev, std::memory_order_relaxed) != (tricky_pointer(predecessor) | FLAG), nullp…  in release()
354                         release_internal_lock(*predecessor);  in release()
361 …__TBB_ASSERT(predecessor && predecessor->my_internal_lock.load(std::memory_order_relaxed)==ACQUIRE…  in release()
362                 tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed);  in release()
365                 tricky_pointer::store(predecessor->my_next, nullptr, std::memory_order_release);  in release()
368 …rder_acquire) && !s.my_mutex->q_tail.compare_exchange_strong(expected, predecessor, std::memory_or…  in release()
376 …           tmp = tricky_pointer::exchange(l_next->my_prev, predecessor, std::memory_order_release);  in release()
378 …    __TBB_ASSERT(tricky_pointer::load(s.my_prev, std::memory_order_relaxed)==predecessor, nullptr);  in release()
379 …  predecessor->my_next.store(s.my_next.load(std::memory_order_relaxed), std::memory_order_release);  in release()
382                 release_internal_lock(*predecessor);  in release()
525         queuing_rw_mutex::scoped_lock * predecessor;  in upgrade_to_writer()  local
527         predecessor = tricky_pointer::fetch_add(s.my_prev, FLAG, std::memory_order_acquire);  in upgrade_to_writer()
528         if( predecessor ) {  in upgrade_to_writer()
529             bool success = try_acquire_internal_lock(*predecessor);  in upgrade_to_writer()
536 …predecessor->my_state.compare_exchange_strong(old_state, STATE_UPGRADE_WAITING, std::memory_order_…  in upgrade_to_writer()
541 …y_pointer::compare_exchange_strong(s.my_prev, tricky_pointer(predecessor)|FLAG, predecessor, std::…  in upgrade_to_writer()
543                     tricky_pointer::spin_wait_while_eq(s.my_prev, predecessor);  in upgrade_to_writer()
544                     predecessor = tricky_pointer::load(s.my_prev, std::memory_order_relaxed);  in upgrade_to_writer()
547                     tricky_pointer::spin_wait_while_eq(s.my_prev, tricky_pointer(predecessor)|FLAG);  in upgrade_to_writer()
548                     release_internal_lock(*predecessor);  in upgrade_to_writer()
551                 tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed);  in upgrade_to_writer()
552                 release_internal_lock(*predecessor);  in upgrade_to_writer()
553                 tricky_pointer::spin_wait_while_eq(s.my_prev, predecessor);  in upgrade_to_writer()
554                 predecessor = tricky_pointer::load(s.my_prev, std::memory_order_relaxed);  in upgrade_to_writer()
556             if( predecessor )  in upgrade_to_writer()
561         __TBB_ASSERT( !predecessor && !s.my_prev, nullptr );  in upgrade_to_writer()