1 //===-- msan_allocator.cc --------------------------- ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file is a part of MemorySanitizer. 11 // 12 // MemorySanitizer allocator. 13 //===----------------------------------------------------------------------===// 14 15 #include "sanitizer_common/sanitizer_allocator.h" 16 #include "sanitizer_common/sanitizer_allocator_interface.h" 17 #include "msan.h" 18 #include "msan_allocator.h" 19 #include "msan_origin.h" 20 #include "msan_thread.h" 21 22 namespace __msan { 23 24 struct Metadata { 25 uptr requested_size; 26 }; 27 28 struct MsanMapUnmapCallback { 29 void OnMap(uptr p, uptr size) const {} 30 void OnUnmap(uptr p, uptr size) const { 31 __msan_unpoison((void *)p, size); 32 33 // We are about to unmap a chunk of user memory. 34 // Mark the corresponding shadow memory as not needed. 35 FlushUnneededShadowMemory(MEM_TO_SHADOW(p), size); 36 if (__msan_get_track_origins()) 37 FlushUnneededShadowMemory(MEM_TO_ORIGIN(p), size); 38 } 39 }; 40 41 #if defined(__mips64) 42 static const uptr kMaxAllowedMallocSize = 2UL << 30; 43 static const uptr kRegionSizeLog = 20; 44 static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; 45 typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; 46 typedef CompactSizeClassMap SizeClassMap; 47 48 typedef SizeClassAllocator32<0, SANITIZER_MMAP_RANGE_SIZE, sizeof(Metadata), 49 SizeClassMap, kRegionSizeLog, ByteMap, 50 MsanMapUnmapCallback> PrimaryAllocator; 51 #elif defined(__x86_64__) 52 static const uptr kAllocatorSpace = 0x600000000000ULL; 53 static const uptr kAllocatorSize = 0x80000000000; // 8T. 54 static const uptr kMetadataSize = sizeof(Metadata); 55 static const uptr kMaxAllowedMallocSize = 8UL << 30; 56 57 typedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, kMetadataSize, 58 DefaultSizeClassMap, 59 MsanMapUnmapCallback> PrimaryAllocator; 60 #endif 61 typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; 62 typedef LargeMmapAllocator<MsanMapUnmapCallback> SecondaryAllocator; 63 typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, 64 SecondaryAllocator> Allocator; 65 66 static Allocator allocator; 67 static AllocatorCache fallback_allocator_cache; 68 static SpinMutex fallback_mutex; 69 70 static int inited = 0; 71 72 static inline void Init() { 73 if (inited) return; 74 __msan_init(); 75 inited = true; // this must happen before any threads are created. 76 allocator.Init(); 77 } 78 79 AllocatorCache *GetAllocatorCache(MsanThreadLocalMallocStorage *ms) { 80 CHECK(ms); 81 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache)); 82 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache); 83 } 84 85 void MsanThreadLocalMallocStorage::CommitBack() { 86 allocator.SwallowCache(GetAllocatorCache(this)); 87 } 88 89 static void *MsanAllocate(StackTrace *stack, uptr size, uptr alignment, 90 bool zeroise) { 91 Init(); 92 if (size > kMaxAllowedMallocSize) { 93 Report("WARNING: MemorySanitizer failed to allocate %p bytes\n", 94 (void *)size); 95 return AllocatorReturnNull(); 96 } 97 MsanThread *t = GetCurrentThread(); 98 void *allocated; 99 if (t) { 100 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); 101 allocated = allocator.Allocate(cache, size, alignment, false); 102 } else { 103 SpinMutexLock l(&fallback_mutex); 104 AllocatorCache *cache = &fallback_allocator_cache; 105 allocated = allocator.Allocate(cache, size, alignment, false); 106 } 107 Metadata *meta = 108 reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated)); 109 meta->requested_size = size; 110 if (zeroise) { 111 __msan_clear_and_unpoison(allocated, size); 112 } else if (flags()->poison_in_malloc) { 113 __msan_poison(allocated, size); 114 if (__msan_get_track_origins()) { 115 Origin o = Origin::CreateHeapOrigin(stack); 116 __msan_set_origin(allocated, size, o.raw_id()); 117 } 118 } 119 MSAN_MALLOC_HOOK(allocated, size); 120 return allocated; 121 } 122 123 void MsanDeallocate(StackTrace *stack, void *p) { 124 CHECK(p); 125 Init(); 126 MSAN_FREE_HOOK(p); 127 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p)); 128 uptr size = meta->requested_size; 129 meta->requested_size = 0; 130 // This memory will not be reused by anyone else, so we are free to keep it 131 // poisoned. 132 if (flags()->poison_in_free) { 133 __msan_poison(p, size); 134 if (__msan_get_track_origins()) { 135 Origin o = Origin::CreateHeapOrigin(stack); 136 __msan_set_origin(p, size, o.raw_id()); 137 } 138 } 139 MsanThread *t = GetCurrentThread(); 140 if (t) { 141 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); 142 allocator.Deallocate(cache, p); 143 } else { 144 SpinMutexLock l(&fallback_mutex); 145 AllocatorCache *cache = &fallback_allocator_cache; 146 allocator.Deallocate(cache, p); 147 } 148 } 149 150 void *MsanReallocate(StackTrace *stack, void *old_p, uptr new_size, 151 uptr alignment, bool zeroise) { 152 if (!old_p) 153 return MsanAllocate(stack, new_size, alignment, zeroise); 154 if (!new_size) { 155 MsanDeallocate(stack, old_p); 156 return 0; 157 } 158 Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p)); 159 uptr old_size = meta->requested_size; 160 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p); 161 if (new_size <= actually_allocated_size) { 162 // We are not reallocating here. 163 meta->requested_size = new_size; 164 if (new_size > old_size) 165 __msan_poison((char*)old_p + old_size, new_size - old_size); 166 return old_p; 167 } 168 uptr memcpy_size = Min(new_size, old_size); 169 void *new_p = MsanAllocate(stack, new_size, alignment, zeroise); 170 // Printf("realloc: old_size %zd new_size %zd\n", old_size, new_size); 171 if (new_p) { 172 __msan_memcpy(new_p, old_p, memcpy_size); 173 MsanDeallocate(stack, old_p); 174 } 175 return new_p; 176 } 177 178 static uptr AllocationSize(const void *p) { 179 if (p == 0) return 0; 180 const void *beg = allocator.GetBlockBegin(p); 181 if (beg != p) return 0; 182 Metadata *b = (Metadata *)allocator.GetMetaData(p); 183 return b->requested_size; 184 } 185 186 } // namespace __msan 187 188 using namespace __msan; 189 190 uptr __sanitizer_get_current_allocated_bytes() { 191 uptr stats[AllocatorStatCount]; 192 allocator.GetStats(stats); 193 return stats[AllocatorStatAllocated]; 194 } 195 196 uptr __sanitizer_get_heap_size() { 197 uptr stats[AllocatorStatCount]; 198 allocator.GetStats(stats); 199 return stats[AllocatorStatMapped]; 200 } 201 202 uptr __sanitizer_get_free_bytes() { return 1; } 203 204 uptr __sanitizer_get_unmapped_bytes() { return 1; } 205 206 uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } 207 208 int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; } 209 210 uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); } 211