1 //===- llvm/unittest/Support/MemoryBufferTest.cpp - MemoryBuffer tests ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements unit tests for the MemoryBuffer support class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/Support/MemoryBuffer.h" 14 #include "llvm/ADT/ScopeExit.h" 15 #include "llvm/Support/FileSystem.h" 16 #include "llvm/Support/FileUtilities.h" 17 #include "llvm/Support/Process.h" 18 #include "llvm/Support/raw_ostream.h" 19 #include "llvm/Testing/Support/Error.h" 20 #include "gtest/gtest.h" 21 #if LLVM_ENABLE_THREADS 22 #include <thread> 23 #endif 24 #if LLVM_ON_UNIX 25 #include <unistd.h> 26 #endif 27 #if _WIN32 28 #include <windows.h> 29 #endif 30 31 using namespace llvm; 32 33 #define ASSERT_NO_ERROR(x) \ 34 if (std::error_code ASSERT_NO_ERROR_ec = x) { \ 35 SmallString<128> MessageStorage; \ 36 raw_svector_ostream Message(MessageStorage); \ 37 Message << #x ": did not return errc::success.\n" \ 38 << "error number: " << ASSERT_NO_ERROR_ec.value() << "\n" \ 39 << "error message: " << ASSERT_NO_ERROR_ec.message() << "\n"; \ 40 GTEST_FATAL_FAILURE_(MessageStorage.c_str()); \ 41 } else { \ 42 } 43 44 #define ASSERT_ERROR(x) \ 45 if (!x) { \ 46 SmallString<128> MessageStorage; \ 47 raw_svector_ostream Message(MessageStorage); \ 48 Message << #x ": did not return a failure error code.\n"; \ 49 GTEST_FATAL_FAILURE_(MessageStorage.c_str()); \ 50 } 51 52 namespace { 53 54 class MemoryBufferTest : public testing::Test { 55 protected: 56 MemoryBufferTest() 57 : data("this is some data") 58 { } 59 60 void SetUp() override {} 61 62 /// Common testing for different modes of getOpenFileSlice. 63 /// Creates a temporary file with known contents, and uses 64 /// MemoryBuffer::getOpenFileSlice to map it. 65 /// If \p Reopen is true, the file is closed after creating and reopened 66 /// anew before using MemoryBuffer. 67 void testGetOpenFileSlice(bool Reopen); 68 69 typedef std::unique_ptr<MemoryBuffer> OwningBuffer; 70 71 std::string data; 72 }; 73 74 TEST_F(MemoryBufferTest, get) { 75 // Default name and null-terminator flag 76 OwningBuffer MB1(MemoryBuffer::getMemBuffer(data)); 77 EXPECT_TRUE(nullptr != MB1.get()); 78 79 // RequiresNullTerminator = false 80 OwningBuffer MB2(MemoryBuffer::getMemBuffer(data, "one", false)); 81 EXPECT_TRUE(nullptr != MB2.get()); 82 83 // RequiresNullTerminator = true 84 OwningBuffer MB3(MemoryBuffer::getMemBuffer(data, "two", true)); 85 EXPECT_TRUE(nullptr != MB3.get()); 86 87 // verify all 3 buffers point to the same address 88 EXPECT_EQ(MB1->getBufferStart(), MB2->getBufferStart()); 89 EXPECT_EQ(MB2->getBufferStart(), MB3->getBufferStart()); 90 91 // verify the original data is unmodified after deleting the buffers 92 MB1.reset(); 93 MB2.reset(); 94 MB3.reset(); 95 EXPECT_EQ("this is some data", data); 96 } 97 98 TEST_F(MemoryBufferTest, getOpenFile) { 99 int FD; 100 SmallString<64> TestPath; 101 ASSERT_EQ(sys::fs::createTemporaryFile("MemoryBufferTest_getOpenFile", "temp", 102 FD, TestPath), 103 std::error_code()); 104 105 FileRemover Cleanup(TestPath); 106 raw_fd_ostream OF(FD, /*shouldClose*/ true); 107 OF << "12345678"; 108 OF.close(); 109 110 { 111 Expected<sys::fs::file_t> File = sys::fs::openNativeFileForRead(TestPath); 112 ASSERT_THAT_EXPECTED(File, Succeeded()); 113 auto OnExit = 114 make_scope_exit([&] { ASSERT_NO_ERROR(sys::fs::closeFile(*File)); }); 115 ErrorOr<OwningBuffer> MB = MemoryBuffer::getOpenFile(*File, TestPath, 6); 116 ASSERT_NO_ERROR(MB.getError()); 117 EXPECT_EQ("123456", MB.get()->getBuffer()); 118 } 119 { 120 Expected<sys::fs::file_t> File = sys::fs::openNativeFileForWrite( 121 TestPath, sys::fs::CD_OpenExisting, sys::fs::OF_None); 122 ASSERT_THAT_EXPECTED(File, Succeeded()); 123 auto OnExit = 124 make_scope_exit([&] { ASSERT_NO_ERROR(sys::fs::closeFile(*File)); }); 125 ASSERT_ERROR(MemoryBuffer::getOpenFile(*File, TestPath, 6).getError()); 126 } 127 } 128 129 TEST_F(MemoryBufferTest, NullTerminator4K) { 130 // Test that a file with size that is a multiple of the page size can be null 131 // terminated correctly by MemoryBuffer. 132 int TestFD; 133 SmallString<64> TestPath; 134 sys::fs::createTemporaryFile("MemoryBufferTest_NullTerminator4K", "temp", 135 TestFD, TestPath); 136 FileRemover Cleanup(TestPath); 137 raw_fd_ostream OF(TestFD, true, /*unbuffered=*/true); 138 for (unsigned i = 0; i < 4096 / 16; ++i) { 139 OF << "0123456789abcdef"; 140 } 141 OF.close(); 142 143 ErrorOr<OwningBuffer> MB = MemoryBuffer::getFile(TestPath.c_str()); 144 std::error_code EC = MB.getError(); 145 ASSERT_FALSE(EC); 146 147 const char *BufData = MB.get()->getBufferStart(); 148 EXPECT_EQ('f', BufData[4095]); 149 EXPECT_EQ('\0', BufData[4096]); 150 } 151 152 TEST_F(MemoryBufferTest, copy) { 153 // copy with no name 154 OwningBuffer MBC1(MemoryBuffer::getMemBufferCopy(data)); 155 EXPECT_TRUE(nullptr != MBC1.get()); 156 157 // copy with a name 158 OwningBuffer MBC2(MemoryBuffer::getMemBufferCopy(data, "copy")); 159 EXPECT_TRUE(nullptr != MBC2.get()); 160 161 // verify the two copies do not point to the same place 162 EXPECT_NE(MBC1->getBufferStart(), MBC2->getBufferStart()); 163 } 164 165 #if LLVM_ENABLE_THREADS 166 TEST_F(MemoryBufferTest, createFromPipe) { 167 sys::fs::file_t pipes[2]; 168 #if LLVM_ON_UNIX 169 ASSERT_EQ(::pipe(pipes), 0) << strerror(errno); 170 #else 171 ASSERT_TRUE(::CreatePipe(&pipes[0], &pipes[1], nullptr, 0)) 172 << ::GetLastError(); 173 #endif 174 auto ReadCloser = make_scope_exit([&] { sys::fs::closeFile(pipes[0]); }); 175 std::thread Writer([&] { 176 auto WriteCloser = make_scope_exit([&] { sys::fs::closeFile(pipes[1]); }); 177 for (unsigned i = 0; i < 5; ++i) { 178 std::this_thread::sleep_for(std::chrono::milliseconds(10)); 179 #if LLVM_ON_UNIX 180 ASSERT_EQ(::write(pipes[1], "foo", 3), 3) << strerror(errno); 181 #else 182 DWORD Written; 183 ASSERT_TRUE(::WriteFile(pipes[1], "foo", 3, &Written, nullptr)) 184 << ::GetLastError(); 185 ASSERT_EQ(Written, 3u); 186 #endif 187 } 188 }); 189 ErrorOr<OwningBuffer> MB = 190 MemoryBuffer::getOpenFile(pipes[0], "pipe", /*FileSize*/ -1); 191 Writer.join(); 192 ASSERT_NO_ERROR(MB.getError()); 193 EXPECT_EQ(MB.get()->getBuffer(), "foofoofoofoofoo"); 194 } 195 #endif 196 197 TEST_F(MemoryBufferTest, make_new) { 198 // 0-sized buffer 199 OwningBuffer Zero(WritableMemoryBuffer::getNewUninitMemBuffer(0)); 200 EXPECT_TRUE(nullptr != Zero.get()); 201 202 // uninitialized buffer with no name 203 OwningBuffer One(WritableMemoryBuffer::getNewUninitMemBuffer(321)); 204 EXPECT_TRUE(nullptr != One.get()); 205 206 // uninitialized buffer with name 207 OwningBuffer Two(WritableMemoryBuffer::getNewUninitMemBuffer(123, "bla")); 208 EXPECT_TRUE(nullptr != Two.get()); 209 210 // 0-initialized buffer with no name 211 OwningBuffer Three(WritableMemoryBuffer::getNewMemBuffer(321, data)); 212 EXPECT_TRUE(nullptr != Three.get()); 213 for (size_t i = 0; i < 321; ++i) 214 EXPECT_EQ(0, Three->getBufferStart()[0]); 215 216 // 0-initialized buffer with name 217 OwningBuffer Four(WritableMemoryBuffer::getNewMemBuffer(123, "zeros")); 218 EXPECT_TRUE(nullptr != Four.get()); 219 for (size_t i = 0; i < 123; ++i) 220 EXPECT_EQ(0, Four->getBufferStart()[0]); 221 } 222 223 void MemoryBufferTest::testGetOpenFileSlice(bool Reopen) { 224 // Test that MemoryBuffer::getOpenFile works properly when no null 225 // terminator is requested and the size is large enough to trigger 226 // the usage of memory mapping. 227 int TestFD; 228 SmallString<64> TestPath; 229 // Create a temporary file and write data into it. 230 sys::fs::createTemporaryFile("prefix", "temp", TestFD, TestPath); 231 FileRemover Cleanup(TestPath); 232 // OF is responsible for closing the file; If the file is not 233 // reopened, it will be unbuffered so that the results are 234 // immediately visible through the fd. 235 raw_fd_ostream OF(TestFD, true, !Reopen); 236 for (int i = 0; i < 60000; ++i) { 237 OF << "0123456789"; 238 } 239 240 if (Reopen) { 241 OF.close(); 242 EXPECT_FALSE(sys::fs::openFileForRead(TestPath.c_str(), TestFD)); 243 } 244 245 ErrorOr<OwningBuffer> Buf = MemoryBuffer::getOpenFileSlice( 246 sys::fs::convertFDToNativeFile(TestFD), TestPath.c_str(), 247 40000, // Size 248 80000 // Offset 249 ); 250 251 std::error_code EC = Buf.getError(); 252 EXPECT_FALSE(EC); 253 254 StringRef BufData = Buf.get()->getBuffer(); 255 EXPECT_EQ(BufData.size(), 40000U); 256 EXPECT_EQ(BufData[0], '0'); 257 EXPECT_EQ(BufData[9], '9'); 258 } 259 260 TEST_F(MemoryBufferTest, getOpenFileNoReopen) { 261 testGetOpenFileSlice(false); 262 } 263 264 TEST_F(MemoryBufferTest, getOpenFileReopened) { 265 testGetOpenFileSlice(true); 266 } 267 268 TEST_F(MemoryBufferTest, reference) { 269 OwningBuffer MB(MemoryBuffer::getMemBuffer(data)); 270 MemoryBufferRef MBR(*MB); 271 272 EXPECT_EQ(MB->getBufferStart(), MBR.getBufferStart()); 273 EXPECT_EQ(MB->getBufferIdentifier(), MBR.getBufferIdentifier()); 274 } 275 276 TEST_F(MemoryBufferTest, slice) { 277 // Create a file that is six pages long with different data on each page. 278 int FD; 279 SmallString<64> TestPath; 280 sys::fs::createTemporaryFile("MemoryBufferTest_Slice", "temp", FD, TestPath); 281 FileRemover Cleanup(TestPath); 282 raw_fd_ostream OF(FD, true, /*unbuffered=*/true); 283 for (unsigned i = 0; i < 0x2000 / 8; ++i) { 284 OF << "12345678"; 285 } 286 for (unsigned i = 0; i < 0x2000 / 8; ++i) { 287 OF << "abcdefgh"; 288 } 289 for (unsigned i = 0; i < 0x2000 / 8; ++i) { 290 OF << "ABCDEFGH"; 291 } 292 OF.close(); 293 294 // Try offset of one page. 295 ErrorOr<OwningBuffer> MB = MemoryBuffer::getFileSlice(TestPath.str(), 296 0x4000, 0x1000); 297 std::error_code EC = MB.getError(); 298 ASSERT_FALSE(EC); 299 EXPECT_EQ(0x4000UL, MB.get()->getBufferSize()); 300 301 StringRef BufData = MB.get()->getBuffer(); 302 EXPECT_TRUE(BufData.substr(0x0000,8).equals("12345678")); 303 EXPECT_TRUE(BufData.substr(0x0FF8,8).equals("12345678")); 304 EXPECT_TRUE(BufData.substr(0x1000,8).equals("abcdefgh")); 305 EXPECT_TRUE(BufData.substr(0x2FF8,8).equals("abcdefgh")); 306 EXPECT_TRUE(BufData.substr(0x3000,8).equals("ABCDEFGH")); 307 EXPECT_TRUE(BufData.substr(0x3FF8,8).equals("ABCDEFGH")); 308 309 // Try non-page aligned. 310 ErrorOr<OwningBuffer> MB2 = MemoryBuffer::getFileSlice(TestPath.str(), 311 0x3000, 0x0800); 312 EC = MB2.getError(); 313 ASSERT_FALSE(EC); 314 EXPECT_EQ(0x3000UL, MB2.get()->getBufferSize()); 315 316 StringRef BufData2 = MB2.get()->getBuffer(); 317 EXPECT_TRUE(BufData2.substr(0x0000,8).equals("12345678")); 318 EXPECT_TRUE(BufData2.substr(0x17F8,8).equals("12345678")); 319 EXPECT_TRUE(BufData2.substr(0x1800,8).equals("abcdefgh")); 320 EXPECT_TRUE(BufData2.substr(0x2FF8,8).equals("abcdefgh")); 321 } 322 323 TEST_F(MemoryBufferTest, writableSlice) { 324 // Create a file initialized with some data 325 int FD; 326 SmallString<64> TestPath; 327 sys::fs::createTemporaryFile("MemoryBufferTest_WritableSlice", "temp", FD, 328 TestPath); 329 FileRemover Cleanup(TestPath); 330 raw_fd_ostream OF(FD, true); 331 for (unsigned i = 0; i < 0x1000; ++i) 332 OF << "0123456789abcdef"; 333 OF.close(); 334 335 { 336 auto MBOrError = 337 WritableMemoryBuffer::getFileSlice(TestPath.str(), 0x6000, 0x2000); 338 ASSERT_FALSE(MBOrError.getError()); 339 // Write some data. It should be mapped private, so that upon completion 340 // the original file contents are not modified. 341 WritableMemoryBuffer &MB = **MBOrError; 342 ASSERT_EQ(0x6000u, MB.getBufferSize()); 343 char *Start = MB.getBufferStart(); 344 ASSERT_EQ(MB.getBufferEnd(), MB.getBufferStart() + MB.getBufferSize()); 345 ::memset(Start, 'x', MB.getBufferSize()); 346 } 347 348 auto MBOrError = MemoryBuffer::getFile(TestPath); 349 ASSERT_FALSE(MBOrError.getError()); 350 auto &MB = **MBOrError; 351 ASSERT_EQ(0x10000u, MB.getBufferSize()); 352 for (size_t i = 0; i < MB.getBufferSize(); i += 0x10) 353 EXPECT_EQ("0123456789abcdef", MB.getBuffer().substr(i, 0x10)) << "i: " << i; 354 } 355 356 TEST_F(MemoryBufferTest, writeThroughFile) { 357 // Create a file initialized with some data 358 int FD; 359 SmallString<64> TestPath; 360 sys::fs::createTemporaryFile("MemoryBufferTest_WriteThrough", "temp", FD, 361 TestPath); 362 FileRemover Cleanup(TestPath); 363 raw_fd_ostream OF(FD, true); 364 OF << "0123456789abcdef"; 365 OF.close(); 366 { 367 auto MBOrError = WriteThroughMemoryBuffer::getFile(TestPath); 368 ASSERT_FALSE(MBOrError.getError()); 369 // Write some data. It should be mapped readwrite, so that upon completion 370 // the original file contents are modified. 371 WriteThroughMemoryBuffer &MB = **MBOrError; 372 ASSERT_EQ(16u, MB.getBufferSize()); 373 char *Start = MB.getBufferStart(); 374 ASSERT_EQ(MB.getBufferEnd(), MB.getBufferStart() + MB.getBufferSize()); 375 ::memset(Start, 'x', MB.getBufferSize()); 376 } 377 378 auto MBOrError = MemoryBuffer::getFile(TestPath); 379 ASSERT_FALSE(MBOrError.getError()); 380 auto &MB = **MBOrError; 381 ASSERT_EQ(16u, MB.getBufferSize()); 382 EXPECT_EQ("xxxxxxxxxxxxxxxx", MB.getBuffer()); 383 } 384 385 TEST_F(MemoryBufferTest, mmapVolatileNoNull) { 386 // Verify that `MemoryBuffer::getOpenFile` will use mmap when 387 // `RequiresNullTerminator = false`, `IsVolatile = true`, and the file is 388 // large enough to use mmap. 389 // 390 // This is done because Clang should use this mode to open module files, and 391 // falling back to malloc for them causes a huge memory usage increase. 392 393 int FD; 394 SmallString<64> TestPath; 395 ASSERT_NO_ERROR(sys::fs::createTemporaryFile( 396 "MemoryBufferTest_mmapVolatileNoNull", "temp", FD, TestPath)); 397 FileRemover Cleanup(TestPath); 398 raw_fd_ostream OF(FD, true); 399 // Create a file large enough to mmap. 4 pages should be enough. 400 unsigned PageSize = sys::Process::getPageSizeEstimate(); 401 unsigned FileWrites = (PageSize * 4) / 8; 402 for (unsigned i = 0; i < FileWrites; ++i) 403 OF << "01234567"; 404 OF.close(); 405 406 Expected<sys::fs::file_t> File = sys::fs::openNativeFileForRead(TestPath); 407 ASSERT_THAT_EXPECTED(File, Succeeded()); 408 auto OnExit = 409 make_scope_exit([&] { ASSERT_NO_ERROR(sys::fs::closeFile(*File)); }); 410 411 auto MBOrError = MemoryBuffer::getOpenFile(*File, TestPath, 412 /*FileSize=*/-1, /*RequiresNullTerminator=*/false, /*IsVolatile=*/true); 413 ASSERT_NO_ERROR(MBOrError.getError()) 414 OwningBuffer MB = std::move(*MBOrError); 415 EXPECT_EQ(MB->getBufferKind(), MemoryBuffer::MemoryBuffer_MMap); 416 EXPECT_EQ(MB->getBufferSize(), std::size_t(FileWrites * 8)); 417 EXPECT_TRUE(MB->getBuffer().startswith("01234567")); 418 } 419 } 420