1 /* 2 Copyright (c) 2005-2021 Intel Corporation 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 #include "oneapi/tbb/detail/_assert.h" 18 #include "oneapi/tbb/detail/_rtm_mutex.h" 19 #include "itt_notify.h" 20 #include "governor.h" 21 #include "misc.h" 22 23 #include <atomic> 24 25 namespace tbb { 26 namespace detail { 27 namespace r1 { 28 29 // maximum number of times to retry 30 // TODO: experiment on retry values. 31 static constexpr int retry_threshold = 10; 32 33 struct rtm_mutex_impl { 34 //! Release speculative mutex 35 static void release(d1::rtm_mutex::scoped_lock& s) { 36 switch(s.m_transaction_state) { 37 case d1::rtm_mutex::rtm_state::rtm_transacting: 38 __TBB_ASSERT(is_in_transaction(), "m_transaction_state && not speculating"); 39 end_transaction(); 40 s.m_mutex = nullptr; 41 break; 42 case d1::rtm_mutex::rtm_state::rtm_real: 43 s.m_mutex->unlock(); 44 s.m_mutex = nullptr; 45 break; 46 case d1::rtm_mutex::rtm_state::rtm_none: 47 __TBB_ASSERT(false, "mutex is not locked, but in release"); 48 break; 49 default: 50 __TBB_ASSERT(false, "invalid m_transaction_state"); 51 } 52 s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_none; 53 } 54 55 //! Acquire lock on the given mutex. 56 static void acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s, bool only_speculate) { 57 __TBB_ASSERT(s.m_transaction_state == d1::rtm_mutex::rtm_state::rtm_none, "scoped_lock already in transaction"); 58 if(governor::speculation_enabled()) { 59 int num_retries = 0; 60 unsigned int abort_code = 0; 61 do { 62 if(m.m_flag.load(std::memory_order_acquire)) { 63 if(only_speculate) return; 64 spin_wait_while_eq(m.m_flag, true); 65 } 66 // _xbegin returns -1 on success or the abort code, so capture it 67 if((abort_code = begin_transaction()) == speculation_successful_begin) 68 { 69 // started speculation 70 if(m.m_flag.load(std::memory_order_relaxed)) { 71 abort_transaction(); 72 } 73 s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_transacting; 74 // Don not wrap the following assignment to a function, 75 // because it can abort the transaction in debug. Need mutex for release(). 76 s.m_mutex = &m; 77 return; // successfully started speculation 78 } 79 ++num_retries; 80 } while((abort_code & speculation_retry) != 0 && (num_retries < retry_threshold)); 81 } 82 83 if(only_speculate) return; 84 s.m_mutex = &m; 85 s.m_mutex->lock(); 86 s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_real; 87 return; 88 } 89 90 //! Try to acquire lock on the given mutex. 91 static bool try_acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s) { 92 acquire(m, s, /*only_speculate=*/true); 93 if (s.m_transaction_state == d1::rtm_mutex::rtm_state::rtm_transacting) { 94 return true; 95 } 96 __TBB_ASSERT(s.m_transaction_state == d1::rtm_mutex::rtm_state::rtm_none, NULL); 97 // transacting acquire failed. try_lock the real mutex 98 if (m.try_lock()) { 99 s.m_mutex = &m; 100 s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_real; 101 return true; 102 } 103 return false; 104 } 105 }; 106 107 void __TBB_EXPORTED_FUNC acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s, bool only_speculate) { 108 rtm_mutex_impl::acquire(m, s, only_speculate); 109 } 110 bool __TBB_EXPORTED_FUNC try_acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s) { 111 return rtm_mutex_impl::try_acquire(m, s); 112 } 113 void __TBB_EXPORTED_FUNC release(d1::rtm_mutex::scoped_lock& s) { 114 rtm_mutex_impl::release(s); 115 } 116 117 } // namespace r1 118 } // namespace detail 119 } // namespace tbb 120 121