1 /* 2 Copyright (c) 2005-2021 Intel Corporation 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 #ifndef __TBB_detail__machine_H 18 #define __TBB_detail__machine_H 19 20 #include "_config.h" 21 #include "_assert.h" 22 23 #include <atomic> 24 #include <climits> 25 #include <cstdint> 26 #include <cstddef> 27 28 #ifdef _WIN32 29 #include <intrin.h> 30 #ifdef __TBBMALLOC_BUILD 31 #define WIN32_LEAN_AND_MEAN 32 #ifndef NOMINMAX 33 #define NOMINMAX 34 #endif 35 #include <windows.h> // SwitchToThread() 36 #endif 37 #ifdef _MSC_VER 38 #if __TBB_x86_64 || __TBB_x86_32 39 #pragma intrinsic(__rdtsc) 40 #endif 41 #endif 42 #endif 43 #if __TBB_x86_64 || __TBB_x86_32 44 #include <immintrin.h> // _mm_pause 45 #endif 46 #if (_WIN32) 47 #include <float.h> // _control87 48 #endif 49 50 #if __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN 51 #include <sched.h> // sched_yield 52 #else 53 #include <thread> // std::this_thread::yield() 54 #endif 55 56 namespace tbb { 57 namespace detail { 58 inline namespace d0 { 59 60 //-------------------------------------------------------------------------------------------------- 61 // Yield implementation 62 //-------------------------------------------------------------------------------------------------- 63 64 #if __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN 65 static inline void yield() { 66 int err = sched_yield(); 67 __TBB_ASSERT_EX(err == 0, "sched_yield has failed"); 68 } 69 #elif __TBBMALLOC_BUILD && _WIN32 70 // Use Windows API for yield in tbbmalloc to avoid dependency on C++ runtime with some implementations. 71 static inline void yield() { 72 SwitchToThread(); 73 } 74 #else 75 using std::this_thread::yield; 76 #endif 77 78 //-------------------------------------------------------------------------------------------------- 79 // atomic_fence implementation 80 //-------------------------------------------------------------------------------------------------- 81 82 #if _MSC_VER && (__TBB_x86_64 || __TBB_x86_32) 83 #pragma intrinsic(_mm_mfence) 84 #endif 85 86 static inline void atomic_fence(std::memory_order order) { 87 #if _MSC_VER && (__TBB_x86_64 || __TBB_x86_32) 88 if (order == std::memory_order_seq_cst || 89 order == std::memory_order_acq_rel || 90 order == std::memory_order_acquire || 91 order == std::memory_order_release ) 92 { 93 _mm_mfence(); 94 return; 95 } 96 #endif /*_MSC_VER && (__TBB_x86_64 || __TBB_x86_32)*/ 97 std::atomic_thread_fence(order); 98 } 99 100 //-------------------------------------------------------------------------------------------------- 101 // Pause implementation 102 //-------------------------------------------------------------------------------------------------- 103 104 static inline void machine_pause(int32_t delay) { 105 #if __TBB_x86_64 || __TBB_x86_32 106 while (delay-- > 0) { _mm_pause(); } 107 #elif __ARM_ARCH_7A__ || __aarch64__ 108 while (delay-- > 0) { __asm__ __volatile__("yield" ::: "memory"); } 109 #else /* Generic */ 110 (void)delay; // suppress without including _template_helpers.h 111 yield(); 112 #endif 113 } 114 115 //////////////////////////////////////////////////////////////////////////////////////////////////// 116 // tbb::detail::log2() implementation 117 //////////////////////////////////////////////////////////////////////////////////////////////////// 118 // TODO: Use log2p1() function that will be available in C++20 standard 119 120 #if defined(__GNUC__) || defined(__clang__) 121 namespace gnu_builtins { 122 inline uintptr_t clz(unsigned int x) { return __builtin_clz(x); } 123 inline uintptr_t clz(unsigned long int x) { return __builtin_clzl(x); } 124 inline uintptr_t clz(unsigned long long int x) { return __builtin_clzll(x); } 125 } 126 #elif defined(_MSC_VER) 127 #pragma intrinsic(__TBB_W(_BitScanReverse)) 128 namespace msvc_intrinsics { 129 static inline uintptr_t bit_scan_reverse(uintptr_t i) { 130 unsigned long j; 131 __TBB_W(_BitScanReverse)( &j, i ); 132 return j; 133 } 134 } 135 #endif 136 137 template <typename T> 138 constexpr std::uintptr_t number_of_bits() { 139 return sizeof(T) * CHAR_BIT; 140 } 141 142 // logarithm is the index of the most significant non-zero bit 143 static inline uintptr_t machine_log2(uintptr_t x) { 144 #if defined(__GNUC__) || defined(__clang__) 145 // If P is a power of 2 and x<P, then (P-1)-x == (P-1) XOR x 146 return (number_of_bits<decltype(x)>() - 1) ^ gnu_builtins::clz(x); 147 #elif defined(_MSC_VER) 148 return msvc_intrinsics::bit_scan_reverse(x); 149 #elif __i386__ || __i386 /*for Sun OS*/ || __MINGW32__ 150 uintptr_t j, i = x; 151 __asm__("bsr %1,%0" : "=r"(j) : "r"(i)); 152 return j; 153 #elif __powerpc__ || __POWERPC__ 154 #if __TBB_WORDSIZE==8 155 __asm__ __volatile__ ("cntlzd %0,%0" : "+r"(x)); 156 return 63 - static_cast<intptr_t>(x); 157 #else 158 __asm__ __volatile__ ("cntlzw %0,%0" : "+r"(x)); 159 return 31 - static_cast<intptr_t>(x); 160 #endif /*__TBB_WORDSIZE*/ 161 #elif __sparc 162 uint64_t count; 163 // one hot encode 164 x |= (x >> 1); 165 x |= (x >> 2); 166 x |= (x >> 4); 167 x |= (x >> 8); 168 x |= (x >> 16); 169 x |= (x >> 32); 170 // count 1's 171 __asm__ ("popc %1, %0" : "=r"(count) : "r"(x) ); 172 return count - 1; 173 #else 174 intptr_t result = 0; 175 176 if( sizeof(x) > 4 && (uintptr_t tmp = x >> 32) ) { x = tmp; result += 32; } 177 if( uintptr_t tmp = x >> 16 ) { x = tmp; result += 16; } 178 if( uintptr_t tmp = x >> 8 ) { x = tmp; result += 8; } 179 if( uintptr_t tmp = x >> 4 ) { x = tmp; result += 4; } 180 if( uintptr_t tmp = x >> 2 ) { x = tmp; result += 2; } 181 182 return (x & 2) ? result + 1 : result; 183 #endif 184 } 185 186 //////////////////////////////////////////////////////////////////////////////////////////////////// 187 // tbb::detail::reverse_bits() implementation 188 //////////////////////////////////////////////////////////////////////////////////////////////////// 189 #if TBB_USE_CLANG_BITREVERSE_BUILTINS 190 namespace llvm_builtins { 191 inline uint8_t builtin_bitreverse(uint8_t x) { return __builtin_bitreverse8 (x); } 192 inline uint16_t builtin_bitreverse(uint16_t x) { return __builtin_bitreverse16(x); } 193 inline uint32_t builtin_bitreverse(uint32_t x) { return __builtin_bitreverse32(x); } 194 inline uint64_t builtin_bitreverse(uint64_t x) { return __builtin_bitreverse64(x); } 195 } 196 #else // generic 197 template<typename T> 198 struct reverse { 199 static const T byte_table[256]; 200 }; 201 202 template<typename T> 203 const T reverse<T>::byte_table[256] = { 204 0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0, 205 0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8, 206 0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4, 207 0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC, 208 0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2, 209 0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA, 210 0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6, 211 0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE, 212 0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1, 213 0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9, 214 0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5, 215 0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD, 216 0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3, 217 0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB, 218 0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7, 219 0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF 220 }; 221 222 inline unsigned char reverse_byte(unsigned char src) { 223 return reverse<unsigned char>::byte_table[src]; 224 } 225 #endif // TBB_USE_CLANG_BITREVERSE_BUILTINS 226 227 template<typename T> 228 T machine_reverse_bits(T src) { 229 #if TBB_USE_CLANG_BITREVERSE_BUILTINS 230 return builtin_bitreverse(fixed_width_cast(src)); 231 #else /* Generic */ 232 T dst; 233 unsigned char *original = (unsigned char *) &src; 234 unsigned char *reversed = (unsigned char *) &dst; 235 236 for ( int i = sizeof(T) - 1; i >= 0; i-- ) { 237 reversed[i] = reverse_byte( original[sizeof(T) - i - 1] ); 238 } 239 240 return dst; 241 #endif // TBB_USE_CLANG_BITREVERSE_BUILTINS 242 } 243 244 } // inline namespace d0 245 246 namespace d1 { 247 248 #if (_WIN32) 249 // API to retrieve/update FPU control setting 250 #define __TBB_CPU_CTL_ENV_PRESENT 1 251 struct cpu_ctl_env { 252 unsigned int x87cw{}; 253 #if (__TBB_x86_64) 254 // Changing the infinity mode or the floating-point precision is not supported on x64. 255 // The attempt causes an assertion. See 256 // https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/control87-controlfp-control87-2 257 static constexpr unsigned int X87CW_CONTROL_MASK = _MCW_DN | _MCW_EM | _MCW_RC; 258 #else 259 static constexpr unsigned int X87CW_CONTROL_MASK = ~0U; 260 #endif 261 #if (__TBB_x86_32 || __TBB_x86_64) 262 unsigned int mxcsr{}; 263 static constexpr unsigned int MXCSR_CONTROL_MASK = ~0x3fu; /* all except last six status bits */ 264 #endif 265 266 bool operator!=( const cpu_ctl_env& ctl ) const { 267 return 268 #if (__TBB_x86_32 || __TBB_x86_64) 269 mxcsr != ctl.mxcsr || 270 #endif 271 x87cw != ctl.x87cw; 272 } 273 void get_env() { 274 x87cw = _control87(0, 0); 275 #if (__TBB_x86_32 || __TBB_x86_64) 276 mxcsr = _mm_getcsr(); 277 #endif 278 } 279 void set_env() const { 280 _control87(x87cw, X87CW_CONTROL_MASK); 281 #if (__TBB_x86_32 || __TBB_x86_64) 282 _mm_setcsr(mxcsr & MXCSR_CONTROL_MASK); 283 #endif 284 } 285 }; 286 #elif (__TBB_x86_32 || __TBB_x86_64) 287 // API to retrieve/update FPU control setting 288 #define __TBB_CPU_CTL_ENV_PRESENT 1 289 struct cpu_ctl_env { 290 int mxcsr{}; 291 short x87cw{}; 292 static const int MXCSR_CONTROL_MASK = ~0x3f; /* all except last six status bits */ 293 294 bool operator!=(const cpu_ctl_env& ctl) const { 295 return mxcsr != ctl.mxcsr || x87cw != ctl.x87cw; 296 } 297 void get_env() { 298 __asm__ __volatile__( 299 "stmxcsr %0\n\t" 300 "fstcw %1" 301 : "=m"(mxcsr), "=m"(x87cw) 302 ); 303 mxcsr &= MXCSR_CONTROL_MASK; 304 } 305 void set_env() const { 306 __asm__ __volatile__( 307 "ldmxcsr %0\n\t" 308 "fldcw %1" 309 : : "m"(mxcsr), "m"(x87cw) 310 ); 311 } 312 }; 313 #endif 314 315 } // namespace d1 316 317 } // namespace detail 318 } // namespace tbb 319 320 #if !__TBB_CPU_CTL_ENV_PRESENT 321 #include <fenv.h> 322 323 #include <cstring> 324 325 namespace tbb { 326 namespace detail { 327 328 namespace r1 { 329 void* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size); 330 void __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p); 331 } // namespace r1 332 333 namespace d1 { 334 335 class cpu_ctl_env { 336 fenv_t *my_fenv_ptr; 337 public: 338 cpu_ctl_env() : my_fenv_ptr(NULL) {} 339 ~cpu_ctl_env() { 340 if ( my_fenv_ptr ) 341 r1::cache_aligned_deallocate( (void*)my_fenv_ptr ); 342 } 343 // It is possible not to copy memory but just to copy pointers but the following issues should be addressed: 344 // 1. The arena lifetime and the context lifetime are independent; 345 // 2. The user is allowed to recapture different FPU settings to context so 'current FPU settings' inside 346 // dispatch loop may become invalid. 347 // But do we really want to improve the fenv implementation? It seems to be better to replace the fenv implementation 348 // with a platform specific implementation. 349 cpu_ctl_env( const cpu_ctl_env &src ) : my_fenv_ptr(NULL) { 350 *this = src; 351 } 352 cpu_ctl_env& operator=( const cpu_ctl_env &src ) { 353 __TBB_ASSERT( src.my_fenv_ptr, NULL ); 354 if ( !my_fenv_ptr ) 355 my_fenv_ptr = (fenv_t*)r1::cache_aligned_allocate(sizeof(fenv_t)); 356 *my_fenv_ptr = *src.my_fenv_ptr; 357 return *this; 358 } 359 bool operator!=( const cpu_ctl_env &ctl ) const { 360 __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." ); 361 __TBB_ASSERT( ctl.my_fenv_ptr, "cpu_ctl_env is not initialized." ); 362 return std::memcmp( (void*)my_fenv_ptr, (void*)ctl.my_fenv_ptr, sizeof(fenv_t) ); 363 } 364 void get_env () { 365 if ( !my_fenv_ptr ) 366 my_fenv_ptr = (fenv_t*)r1::cache_aligned_allocate(sizeof(fenv_t)); 367 fegetenv( my_fenv_ptr ); 368 } 369 const cpu_ctl_env& set_env () const { 370 __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." ); 371 fesetenv( my_fenv_ptr ); 372 return *this; 373 } 374 }; 375 376 } // namespace d1 377 } // namespace detail 378 } // namespace tbb 379 380 #endif /* !__TBB_CPU_CTL_ENV_PRESENT */ 381 382 #endif // __TBB_detail__machine_H 383