1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 #endif /* JEMALLOC_H_TYPES */ 5 /******************************************************************************/ 6 #ifdef JEMALLOC_H_STRUCTS 7 8 #endif /* JEMALLOC_H_STRUCTS */ 9 /******************************************************************************/ 10 #ifdef JEMALLOC_H_EXTERNS 11 12 #define atomic_read_uint64(p) atomic_add_uint64(p, 0) 13 #define atomic_read_uint32(p) atomic_add_uint32(p, 0) 14 15 #if (LG_SIZEOF_PTR == 3) 16 # define atomic_read_z(p) \ 17 (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)0) 18 # define atomic_add_z(p, x) \ 19 (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x) 20 # define atomic_sub_z(p, x) \ 21 (size_t)atomic_sub_uint64((uint64_t *)p, (uint64_t)x) 22 #elif (LG_SIZEOF_PTR == 2) 23 # define atomic_read_z(p) \ 24 (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)0) 25 # define atomic_add_z(p, x) \ 26 (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x) 27 # define atomic_sub_z(p, x) \ 28 (size_t)atomic_sub_uint32((uint32_t *)p, (uint32_t)x) 29 #endif 30 31 #endif /* JEMALLOC_H_EXTERNS */ 32 /******************************************************************************/ 33 #ifdef JEMALLOC_H_INLINES 34 35 #ifndef JEMALLOC_ENABLE_INLINE 36 uint64_t atomic_add_uint64(uint64_t *p, uint64_t x); 37 uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); 38 uint32_t atomic_add_uint32(uint32_t *p, uint32_t x); 39 uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); 40 #endif 41 42 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_)) 43 /******************************************************************************/ 44 /* 64-bit operations. */ 45 #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 46 JEMALLOC_INLINE uint64_t 47 atomic_add_uint64(uint64_t *p, uint64_t x) 48 { 49 50 return (__sync_add_and_fetch(p, x)); 51 } 52 53 JEMALLOC_INLINE uint64_t 54 atomic_sub_uint64(uint64_t *p, uint64_t x) 55 { 56 57 return (__sync_sub_and_fetch(p, x)); 58 } 59 #elif (defined(JEMALLOC_OSATOMIC)) 60 JEMALLOC_INLINE uint64_t 61 atomic_add_uint64(uint64_t *p, uint64_t x) 62 { 63 64 return (OSAtomicAdd64((int64_t)x, (int64_t *)p)); 65 } 66 67 JEMALLOC_INLINE uint64_t 68 atomic_sub_uint64(uint64_t *p, uint64_t x) 69 { 70 71 return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); 72 } 73 #elif (defined(__amd64_) || defined(__x86_64__)) 74 JEMALLOC_INLINE uint64_t 75 atomic_add_uint64(uint64_t *p, uint64_t x) 76 { 77 78 asm volatile ( 79 "lock; xaddq %0, %1;" 80 : "+r" (x), "=m" (*p) /* Outputs. */ 81 : "m" (*p) /* Inputs. */ 82 ); 83 84 return (x); 85 } 86 87 JEMALLOC_INLINE uint64_t 88 atomic_sub_uint64(uint64_t *p, uint64_t x) 89 { 90 91 x = (uint64_t)(-(int64_t)x); 92 asm volatile ( 93 "lock; xaddq %0, %1;" 94 : "+r" (x), "=m" (*p) /* Outputs. */ 95 : "m" (*p) /* Inputs. */ 96 ); 97 98 return (x); 99 } 100 #else 101 # if (LG_SIZEOF_PTR == 3) 102 # error "Missing implementation for 64-bit atomic operations" 103 # endif 104 #endif 105 106 /******************************************************************************/ 107 /* 32-bit operations. */ 108 #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 109 JEMALLOC_INLINE uint32_t 110 atomic_add_uint32(uint32_t *p, uint32_t x) 111 { 112 113 return (__sync_add_and_fetch(p, x)); 114 } 115 116 JEMALLOC_INLINE uint32_t 117 atomic_sub_uint32(uint32_t *p, uint32_t x) 118 { 119 120 return (__sync_sub_and_fetch(p, x)); 121 } 122 #elif (defined(JEMALLOC_OSATOMIC)) 123 JEMALLOC_INLINE uint32_t 124 atomic_add_uint32(uint32_t *p, uint32_t x) 125 { 126 127 return (OSAtomicAdd32((int32_t)x, (int32_t *)p)); 128 } 129 130 JEMALLOC_INLINE uint32_t 131 atomic_sub_uint32(uint32_t *p, uint32_t x) 132 { 133 134 return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); 135 } 136 #elif (defined(__i386__) || defined(__amd64_) || defined(__x86_64__)) 137 JEMALLOC_INLINE uint32_t 138 atomic_add_uint32(uint32_t *p, uint32_t x) 139 { 140 141 asm volatile ( 142 "lock; xaddl %0, %1;" 143 : "+r" (x), "=m" (*p) /* Outputs. */ 144 : "m" (*p) /* Inputs. */ 145 ); 146 147 return (x); 148 } 149 150 JEMALLOC_INLINE uint32_t 151 atomic_sub_uint32(uint32_t *p, uint32_t x) 152 { 153 154 x = (uint32_t)(-(int32_t)x); 155 asm volatile ( 156 "lock; xaddl %0, %1;" 157 : "+r" (x), "=m" (*p) /* Outputs. */ 158 : "m" (*p) /* Inputs. */ 159 ); 160 161 return (x); 162 } 163 #else 164 # error "Missing implementation for 32-bit atomic operations" 165 #endif 166 #endif 167 168 #endif /* JEMALLOC_H_INLINES */ 169 /******************************************************************************/ 170