1 /* 2 Copyright (c) 2005-2021 Intel Corporation 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 #ifndef __TBB_detail__machine_H 18 #define __TBB_detail__machine_H 19 20 #include "_config.h" 21 #include "_assert.h" 22 23 #include <atomic> 24 #include <climits> 25 #include <cstdint> 26 #include <cstddef> 27 28 #ifdef _MSC_VER 29 #include <intrin.h> 30 #pragma intrinsic(__rdtsc) 31 #endif 32 #if __TBB_x86_64 || __TBB_x86_32 33 #include <immintrin.h> // _mm_pause 34 #endif 35 #if (_WIN32 || _WIN64) 36 #include <float.h> // _control87 37 #endif 38 39 #if __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN 40 #include <sched.h> // sched_yield 41 #else 42 #include <thread> // std::this_thread::yield() 43 #endif 44 45 namespace tbb { 46 namespace detail { 47 inline namespace d0 { 48 49 //-------------------------------------------------------------------------------------------------- 50 // Yield implementation 51 //-------------------------------------------------------------------------------------------------- 52 53 #if __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN 54 static inline void yield() { 55 int err = sched_yield(); 56 __TBB_ASSERT_EX(err == 0, "sched_yiled has failed"); 57 } 58 #else 59 using std::this_thread::yield; 60 #endif 61 62 //-------------------------------------------------------------------------------------------------- 63 // atomic_fence implementation 64 //-------------------------------------------------------------------------------------------------- 65 66 #if (_WIN32 || _WIN64) 67 #pragma intrinsic(_mm_mfence) 68 #endif 69 70 static inline void atomic_fence(std::memory_order order) { 71 #if (_WIN32 || _WIN64) 72 if (order == std::memory_order_seq_cst || 73 order == std::memory_order_acq_rel || 74 order == std::memory_order_acquire || 75 order == std::memory_order_release ) 76 { 77 _mm_mfence(); 78 return; 79 } 80 #endif /*(_WIN32 || _WIN64)*/ 81 std::atomic_thread_fence(order); 82 } 83 84 //-------------------------------------------------------------------------------------------------- 85 // Pause implementation 86 //-------------------------------------------------------------------------------------------------- 87 88 static inline void machine_pause(int32_t delay) { 89 #if __TBB_x86_64 || __TBB_x86_32 90 while (delay-- > 0) { _mm_pause(); } 91 #elif __ARM_ARCH_7A__ || __aarch64__ 92 while (delay-- > 0) { __asm__ __volatile__("yield" ::: "memory"); } 93 #else /* Generic */ 94 (void)delay; // suppress without including _template_helpers.h 95 yield(); 96 #endif 97 } 98 99 //////////////////////////////////////////////////////////////////////////////////////////////////// 100 // tbb::detail::log2() implementation 101 //////////////////////////////////////////////////////////////////////////////////////////////////// 102 // TODO: Use log2p1() function that will be available in C++20 standard 103 104 #if defined(__GNUC__) || defined(__clang__) 105 namespace gnu_builtins { 106 inline uintptr_t clz(unsigned int x) { return __builtin_clz(x); } 107 inline uintptr_t clz(unsigned long int x) { return __builtin_clzl(x); } 108 inline uintptr_t clz(unsigned long long int x) { return __builtin_clzll(x); } 109 } 110 #elif defined(_MSC_VER) 111 #pragma intrinsic(__TBB_W(_BitScanReverse)) 112 namespace msvc_intrinsics { 113 static inline uintptr_t bit_scan_reverse(uintptr_t i) { 114 unsigned long j; 115 __TBB_W(_BitScanReverse)( &j, i ); 116 return j; 117 } 118 } 119 #endif 120 121 template <typename T> 122 constexpr std::uintptr_t number_of_bits() { 123 return sizeof(T) * CHAR_BIT; 124 } 125 126 // logarithm is the index of the most significant non-zero bit 127 static inline uintptr_t machine_log2(uintptr_t x) { 128 #if defined(__GNUC__) || defined(__clang__) 129 // If P is a power of 2 and x<P, then (P-1)-x == (P-1) XOR x 130 return (number_of_bits<decltype(x)>() - 1) ^ gnu_builtins::clz(x); 131 #elif defined(_MSC_VER) 132 return msvc_intrinsics::bit_scan_reverse(x); 133 #elif __i386__ || __i386 /*for Sun OS*/ || __MINGW32__ 134 uintptr_t j, i = x; 135 __asm__("bsr %1,%0" : "=r"(j) : "r"(i)); 136 return j; 137 #elif __powerpc__ || __POWERPC__ 138 #if __TBB_WORDSIZE==8 139 __asm__ __volatile__ ("cntlzd %0,%0" : "+r"(x)); 140 return 63 - static_cast<intptr_t>(x); 141 #else 142 __asm__ __volatile__ ("cntlzw %0,%0" : "+r"(x)); 143 return 31 - static_cast<intptr_t>(x); 144 #endif /*__TBB_WORDSIZE*/ 145 #elif __sparc 146 uint64_t count; 147 // one hot encode 148 x |= (x >> 1); 149 x |= (x >> 2); 150 x |= (x >> 4); 151 x |= (x >> 8); 152 x |= (x >> 16); 153 x |= (x >> 32); 154 // count 1's 155 __asm__ ("popc %1, %0" : "=r"(count) : "r"(x) ); 156 return count - 1; 157 #else 158 intptr_t result = 0; 159 160 if( sizeof(x) > 4 && (uintptr_t tmp = x >> 32) ) { x = tmp; result += 32; } 161 if( uintptr_t tmp = x >> 16 ) { x = tmp; result += 16; } 162 if( uintptr_t tmp = x >> 8 ) { x = tmp; result += 8; } 163 if( uintptr_t tmp = x >> 4 ) { x = tmp; result += 4; } 164 if( uintptr_t tmp = x >> 2 ) { x = tmp; result += 2; } 165 166 return (x & 2) ? result + 1 : result; 167 #endif 168 } 169 170 //////////////////////////////////////////////////////////////////////////////////////////////////// 171 // tbb::detail::reverse_bits() implementation 172 //////////////////////////////////////////////////////////////////////////////////////////////////// 173 #if TBB_USE_CLANG_BITREVERSE_BUILTINS 174 namespace llvm_builtins { 175 inline uint8_t builtin_bitreverse(uint8_t x) { return __builtin_bitreverse8 (x); } 176 inline uint16_t builtin_bitreverse(uint16_t x) { return __builtin_bitreverse16(x); } 177 inline uint32_t builtin_bitreverse(uint32_t x) { return __builtin_bitreverse32(x); } 178 inline uint64_t builtin_bitreverse(uint64_t x) { return __builtin_bitreverse64(x); } 179 } 180 #else // generic 181 template<typename T> 182 struct reverse { 183 static const T byte_table[256]; 184 }; 185 186 template<typename T> 187 const T reverse<T>::byte_table[256] = { 188 0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0, 189 0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8, 190 0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4, 191 0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC, 192 0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2, 193 0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA, 194 0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6, 195 0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE, 196 0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1, 197 0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9, 198 0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5, 199 0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD, 200 0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3, 201 0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB, 202 0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7, 203 0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF 204 }; 205 206 inline unsigned char reverse_byte(unsigned char src) { 207 return reverse<unsigned char>::byte_table[src]; 208 } 209 #endif // TBB_USE_CLANG_BITREVERSE_BUILTINS 210 211 template<typename T> 212 T machine_reverse_bits(T src) { 213 #if TBB_USE_CLANG_BITREVERSE_BUILTINS 214 return builtin_bitreverse(fixed_width_cast(src)); 215 #else /* Generic */ 216 T dst; 217 unsigned char *original = (unsigned char *) &src; 218 unsigned char *reversed = (unsigned char *) &dst; 219 220 for ( int i = sizeof(T) - 1; i >= 0; i-- ) { 221 reversed[i] = reverse_byte( original[sizeof(T) - i - 1] ); 222 } 223 224 return dst; 225 #endif // TBB_USE_CLANG_BITREVERSE_BUILTINS 226 } 227 228 } // inline namespace d0 229 230 namespace d1 { 231 232 #if (_WIN32 || _WIN64) 233 // API to retrieve/update FPU control setting 234 #define __TBB_CPU_CTL_ENV_PRESENT 1 235 struct cpu_ctl_env { 236 unsigned int x87cw{}; 237 #if (__TBB_x86_64) 238 // Changing the infinity mode or the floating-point precision is not supported on x64. 239 // The attempt causes an assertion. See 240 // https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/control87-controlfp-control87-2 241 static constexpr unsigned int X87CW_CONTROL_MASK = _MCW_DN | _MCW_EM | _MCW_RC; 242 #else 243 static constexpr unsigned int X87CW_CONTROL_MASK = ~0U; 244 #endif 245 #if (__TBB_x86_32 || __TBB_x86_64) 246 unsigned int mxcsr{}; 247 static constexpr unsigned int MXCSR_CONTROL_MASK = ~0x3fu; /* all except last six status bits */ 248 #endif 249 250 bool operator!=( const cpu_ctl_env& ctl ) const { 251 return 252 #if (__TBB_x86_32 || __TBB_x86_64) 253 mxcsr != ctl.mxcsr || 254 #endif 255 x87cw != ctl.x87cw; 256 } 257 void get_env() { 258 x87cw = _control87(0, 0); 259 #if (__TBB_x86_32 || __TBB_x86_64) 260 mxcsr = _mm_getcsr(); 261 #endif 262 } 263 void set_env() const { 264 _control87(x87cw, X87CW_CONTROL_MASK); 265 #if (__TBB_x86_32 || __TBB_x86_64) 266 _mm_setcsr(mxcsr & MXCSR_CONTROL_MASK); 267 #endif 268 } 269 }; 270 #elif (__TBB_x86_32 || __TBB_x86_64) 271 // API to retrieve/update FPU control setting 272 #define __TBB_CPU_CTL_ENV_PRESENT 1 273 struct cpu_ctl_env { 274 int mxcsr{}; 275 short x87cw{}; 276 static const int MXCSR_CONTROL_MASK = ~0x3f; /* all except last six status bits */ 277 278 bool operator!=(const cpu_ctl_env& ctl) const { 279 return mxcsr != ctl.mxcsr || x87cw != ctl.x87cw; 280 } 281 void get_env() { 282 __asm__ __volatile__( 283 "stmxcsr %0\n\t" 284 "fstcw %1" 285 : "=m"(mxcsr), "=m"(x87cw) 286 ); 287 mxcsr &= MXCSR_CONTROL_MASK; 288 } 289 void set_env() const { 290 __asm__ __volatile__( 291 "ldmxcsr %0\n\t" 292 "fldcw %1" 293 : : "m"(mxcsr), "m"(x87cw) 294 ); 295 } 296 }; 297 #endif 298 299 } // namespace d1 300 301 } // namespace detail 302 } // namespace tbb 303 304 #if !__TBB_CPU_CTL_ENV_PRESENT 305 #include <fenv.h> 306 307 #include <cstring> 308 309 namespace tbb { 310 namespace detail { 311 312 namespace r1 { 313 void* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size); 314 void __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p); 315 } // namespace r1 316 317 namespace d1 { 318 319 class cpu_ctl_env { 320 fenv_t *my_fenv_ptr; 321 public: 322 cpu_ctl_env() : my_fenv_ptr(NULL) {} 323 ~cpu_ctl_env() { 324 if ( my_fenv_ptr ) 325 r1::cache_aligned_deallocate( (void*)my_fenv_ptr ); 326 } 327 // It is possible not to copy memory but just to copy pointers but the following issues should be addressed: 328 // 1. The arena lifetime and the context lifetime are independent; 329 // 2. The user is allowed to recapture different FPU settings to context so 'current FPU settings' inside 330 // dispatch loop may become invalid. 331 // But do we really want to improve the fenv implementation? It seems to be better to replace the fenv implementation 332 // with a platform specific implementation. 333 cpu_ctl_env( const cpu_ctl_env &src ) : my_fenv_ptr(NULL) { 334 *this = src; 335 } 336 cpu_ctl_env& operator=( const cpu_ctl_env &src ) { 337 __TBB_ASSERT( src.my_fenv_ptr, NULL ); 338 if ( !my_fenv_ptr ) 339 my_fenv_ptr = (fenv_t*)r1::cache_aligned_allocate(sizeof(fenv_t)); 340 *my_fenv_ptr = *src.my_fenv_ptr; 341 return *this; 342 } 343 bool operator!=( const cpu_ctl_env &ctl ) const { 344 __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." ); 345 __TBB_ASSERT( ctl.my_fenv_ptr, "cpu_ctl_env is not initialized." ); 346 return std::memcmp( (void*)my_fenv_ptr, (void*)ctl.my_fenv_ptr, sizeof(fenv_t) ); 347 } 348 void get_env () { 349 if ( !my_fenv_ptr ) 350 my_fenv_ptr = (fenv_t*)r1::cache_aligned_allocate(sizeof(fenv_t)); 351 fegetenv( my_fenv_ptr ); 352 } 353 const cpu_ctl_env& set_env () const { 354 __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." ); 355 fesetenv( my_fenv_ptr ); 356 return *this; 357 } 358 }; 359 360 } // namespace d1 361 } // namespace detail 362 } // namespace tbb 363 364 #endif /* !__TBB_CPU_CTL_ENV_PRESENT */ 365 366 #endif // __TBB_detail__machine_H 367