1 //===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
10 #include "clang/Lex/DependencyDirectivesSourceMinimizer.h"
11 #include "llvm/Support/MemoryBuffer.h"
12 #include "llvm/Support/Threading.h"
13 
14 using namespace clang;
15 using namespace tooling;
16 using namespace dependencies;
17 
18 CachedFileSystemEntry CachedFileSystemEntry::createFileEntry(
19     StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) {
20   // Load the file and its content from the file system.
21   llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile =
22       FS.openFileForRead(Filename);
23   if (!MaybeFile)
24     return MaybeFile.getError();
25   llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status();
26   if (!Stat)
27     return Stat.getError();
28 
29   llvm::vfs::File &F = **MaybeFile;
30   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer =
31       F.getBuffer(Stat->getName());
32   if (!MaybeBuffer)
33     return MaybeBuffer.getError();
34 
35   llvm::SmallString<1024> MinimizedFileContents;
36   // Minimize the file down to directives that might affect the dependencies.
37   const auto &Buffer = *MaybeBuffer;
38   SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens;
39   if (!Minimize || minimizeSourceToDependencyDirectives(
40                        Buffer->getBuffer(), MinimizedFileContents, Tokens)) {
41     // Use the original file unless requested otherwise, or
42     // if the minimization failed.
43     // FIXME: Propage the diagnostic if desired by the client.
44     CachedFileSystemEntry Result;
45     Result.MaybeStat = std::move(*Stat);
46     Result.Contents.reserve(Buffer->getBufferSize() + 1);
47     Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd());
48     // Implicitly null terminate the contents for Clang's lexer.
49     Result.Contents.push_back('\0');
50     Result.Contents.pop_back();
51     return Result;
52   }
53 
54   CachedFileSystemEntry Result;
55   size_t Size = MinimizedFileContents.size();
56   Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(),
57                                        Stat->getLastModificationTime(),
58                                        Stat->getUser(), Stat->getGroup(), Size,
59                                        Stat->getType(), Stat->getPermissions());
60   // The contents produced by the minimizer must be null terminated.
61   assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' &&
62          "not null terminated contents");
63   // Even though there's an implicit null terminator in the minimized contents,
64   // we want to temporarily make it explicit. This will ensure that the
65   // std::move will preserve it even if it needs to do a copy if the
66   // SmallString still has the small capacity.
67   MinimizedFileContents.push_back('\0');
68   Result.Contents = std::move(MinimizedFileContents);
69   // Now make the null terminator implicit again, so that Clang's lexer can find
70   // it right where the buffer ends.
71   Result.Contents.pop_back();
72   return Result;
73 }
74 
75 CachedFileSystemEntry
76 CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) {
77   assert(Stat.isDirectory() && "not a directory!");
78   auto Result = CachedFileSystemEntry();
79   Result.MaybeStat = std::move(Stat);
80   return Result;
81 }
82 
83 DependencyScanningFilesystemSharedCache::
84     DependencyScanningFilesystemSharedCache() {
85   // This heuristic was chosen using a empirical testing on a
86   // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
87   // sharding gives a performance edge by reducing the lock contention.
88   // FIXME: A better heuristic might also consider the OS to account for
89   // the different cost of lock contention on different OSes.
90   NumShards = std::max(2u, llvm::hardware_concurrency() / 4);
91   CacheShards = llvm::make_unique<CacheShard[]>(NumShards);
92 }
93 
94 /// Returns a cache entry for the corresponding key.
95 ///
96 /// A new cache entry is created if the key is not in the cache. This is a
97 /// thread safe call.
98 DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
99 DependencyScanningFilesystemSharedCache::get(StringRef Key) {
100   CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards];
101   std::unique_lock<std::mutex> LockGuard(Shard.CacheLock);
102   auto It = Shard.Cache.try_emplace(Key);
103   return It.first->getValue();
104 }
105 
106 llvm::ErrorOr<llvm::vfs::Status>
107 DependencyScanningWorkerFilesystem::status(const Twine &Path) {
108   SmallString<256> OwnedFilename;
109   StringRef Filename = Path.toStringRef(OwnedFilename);
110 
111   // Check the local cache first.
112   if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename))
113     return Entry->getStatus();
114 
115   // FIXME: Handle PCM/PCH files.
116   // FIXME: Handle module map files.
117 
118   bool KeepOriginalSource = IgnoredFiles.count(Filename);
119   DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
120       &SharedCacheEntry = SharedCache.get(Filename);
121   const CachedFileSystemEntry *Result;
122   {
123     std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
124     CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value;
125 
126     if (!CacheEntry.isValid()) {
127       llvm::vfs::FileSystem &FS = getUnderlyingFS();
128       auto MaybeStatus = FS.status(Filename);
129       if (!MaybeStatus)
130         CacheEntry = CachedFileSystemEntry(MaybeStatus.getError());
131       else if (MaybeStatus->isDirectory())
132         CacheEntry = CachedFileSystemEntry::createDirectoryEntry(
133             std::move(*MaybeStatus));
134       else
135         CacheEntry = CachedFileSystemEntry::createFileEntry(
136             Filename, FS, !KeepOriginalSource);
137     }
138 
139     Result = &CacheEntry;
140   }
141 
142   // Store the result in the local cache.
143   setCachedEntry(Filename, Result);
144   return Result->getStatus();
145 }
146 
147 namespace {
148 
149 /// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
150 /// this subclass.
151 class MinimizedVFSFile final : public llvm::vfs::File {
152 public:
153   MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
154                    llvm::vfs::Status Stat)
155       : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
156 
157   llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
158 
159   const llvm::MemoryBuffer *getBufferPtr() const { return Buffer.get(); }
160 
161   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
162   getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
163             bool IsVolatile) override {
164     return std::move(Buffer);
165   }
166 
167   std::error_code close() override { return {}; }
168 
169 private:
170   std::unique_ptr<llvm::MemoryBuffer> Buffer;
171   llvm::vfs::Status Stat;
172 };
173 
174 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
175 createFile(const CachedFileSystemEntry *Entry) {
176   llvm::ErrorOr<StringRef> Contents = Entry->getContents();
177   if (!Contents)
178     return Contents.getError();
179   return llvm::make_unique<MinimizedVFSFile>(
180       llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(),
181                                        /*RequiresNullTerminator=*/false),
182       *Entry->getStatus());
183 }
184 
185 } // end anonymous namespace
186 
187 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
188 DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
189   SmallString<256> OwnedFilename;
190   StringRef Filename = Path.toStringRef(OwnedFilename);
191 
192   // Check the local cache first.
193   if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename))
194     return createFile(Entry);
195 
196   // FIXME: Handle PCM/PCH files.
197   // FIXME: Handle module map files.
198 
199   bool KeepOriginalSource = IgnoredFiles.count(Filename);
200   DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
201       &SharedCacheEntry = SharedCache.get(Filename);
202   const CachedFileSystemEntry *Result;
203   {
204     std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
205     CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value;
206 
207     if (!CacheEntry.isValid()) {
208       CacheEntry = CachedFileSystemEntry::createFileEntry(
209           Filename, getUnderlyingFS(), !KeepOriginalSource);
210     }
211 
212     Result = &CacheEntry;
213   }
214 
215   // Store the result in the local cache.
216   setCachedEntry(Filename, Result);
217   return createFile(Result);
218 }
219