1 //===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
10 #include "clang/Lex/DependencyDirectivesSourceMinimizer.h"
11 #include "llvm/Support/MemoryBuffer.h"
12 #include "llvm/Support/Threading.h"
13 
14 using namespace clang;
15 using namespace tooling;
16 using namespace dependencies;
17 
18 CachedFileSystemEntry CachedFileSystemEntry::createFileEntry(
19     StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) {
20   // Load the file and its content from the file system.
21   llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile =
22       FS.openFileForRead(Filename);
23   if (!MaybeFile)
24     return MaybeFile.getError();
25   llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status();
26   if (!Stat)
27     return Stat.getError();
28 
29   llvm::vfs::File &F = **MaybeFile;
30   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer =
31       F.getBuffer(Stat->getName());
32   if (!MaybeBuffer)
33     return MaybeBuffer.getError();
34 
35   llvm::SmallString<1024> MinimizedFileContents;
36   // Minimize the file down to directives that might affect the dependencies.
37   const auto &Buffer = *MaybeBuffer;
38   SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens;
39   if (!Minimize || minimizeSourceToDependencyDirectives(
40                        Buffer->getBuffer(), MinimizedFileContents, Tokens)) {
41     // Use the original file unless requested otherwise, or
42     // if the minimization failed.
43     // FIXME: Propage the diagnostic if desired by the client.
44     CachedFileSystemEntry Result;
45     Result.MaybeStat = std::move(*Stat);
46     Result.Contents.reserve(Buffer->getBufferSize() + 1);
47     Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd());
48     // Implicitly null terminate the contents for Clang's lexer.
49     Result.Contents.push_back('\0');
50     Result.Contents.pop_back();
51     return Result;
52   }
53 
54   CachedFileSystemEntry Result;
55   size_t Size = MinimizedFileContents.size();
56   Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(),
57                                        Stat->getLastModificationTime(),
58                                        Stat->getUser(), Stat->getGroup(), Size,
59                                        Stat->getType(), Stat->getPermissions());
60   // The contents produced by the minimizer must be null terminated.
61   assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' &&
62          "not null terminated contents");
63   // Even though there's an implicit null terminator in the minimized contents,
64   // we want to temporarily make it explicit. This will ensure that the
65   // std::move will preserve it even if it needs to do a copy if the
66   // SmallString still has the small capacity.
67   MinimizedFileContents.push_back('\0');
68   Result.Contents = std::move(MinimizedFileContents);
69   // Now make the null terminator implicit again, so that Clang's lexer can find
70   // it right where the buffer ends.
71   Result.Contents.pop_back();
72 
73   // Compute the skipped PP ranges that speedup skipping over inactive
74   // preprocessor blocks.
75   llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32>
76       SkippedRanges;
77   minimize_source_to_dependency_directives::computeSkippedRanges(Tokens,
78                                                                  SkippedRanges);
79   PreprocessorSkippedRangeMapping Mapping;
80   for (const auto &Range : SkippedRanges) {
81     if (Range.Length < 16) {
82       // Ignore small ranges as non-profitable.
83       // FIXME: This is a heuristic, its worth investigating the tradeoffs
84       // when it should be applied.
85       continue;
86     }
87     Mapping[Range.Offset] = Range.Length;
88   }
89   Result.PPSkippedRangeMapping = std::move(Mapping);
90 
91   return Result;
92 }
93 
94 CachedFileSystemEntry
95 CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) {
96   assert(Stat.isDirectory() && "not a directory!");
97   auto Result = CachedFileSystemEntry();
98   Result.MaybeStat = std::move(Stat);
99   return Result;
100 }
101 
102 DependencyScanningFilesystemSharedCache::SingleCache::SingleCache() {
103   // This heuristic was chosen using a empirical testing on a
104   // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
105   // sharding gives a performance edge by reducing the lock contention.
106   // FIXME: A better heuristic might also consider the OS to account for
107   // the different cost of lock contention on different OSes.
108   NumShards =
109       std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4);
110   CacheShards = std::make_unique<CacheShard[]>(NumShards);
111 }
112 
113 DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
114 DependencyScanningFilesystemSharedCache::SingleCache::get(StringRef Key) {
115   CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards];
116   std::unique_lock<std::mutex> LockGuard(Shard.CacheLock);
117   auto It = Shard.Cache.try_emplace(Key);
118   return It.first->getValue();
119 }
120 
121 DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
122 DependencyScanningFilesystemSharedCache::get(StringRef Key, bool Minimized) {
123   SingleCache &Cache = Minimized ? CacheMinimized : CacheOriginal;
124   return Cache.get(Key);
125 }
126 
127 /// Whitelist file extensions that should be minimized, treating no extension as
128 /// a source file that should be minimized.
129 ///
130 /// This is kinda hacky, it would be better if we knew what kind of file Clang
131 /// was expecting instead.
132 static bool shouldMinimizeBasedOnExtension(StringRef Filename) {
133   StringRef Ext = llvm::sys::path::extension(Filename);
134   if (Ext.empty())
135     return true; // C++ standard library
136   return llvm::StringSwitch<bool>(Ext)
137     .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
138     .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
139     .CasesLower(".m", ".mm", true)
140     .CasesLower(".i", ".ii", ".mi", ".mmi", true)
141     .CasesLower(".def", ".inc", true)
142     .Default(false);
143 }
144 
145 
146 static bool shouldCacheStatFailures(StringRef Filename) {
147   StringRef Ext = llvm::sys::path::extension(Filename);
148   if (Ext.empty())
149     return false; // This may be the module cache directory.
150   // Only cache stat failures on source files.
151   return shouldMinimizeBasedOnExtension(Filename);
152 }
153 
154 void DependencyScanningWorkerFilesystem::disableMinimization(
155     StringRef RawFilename) {
156   llvm::SmallString<256> Filename;
157   llvm::sys::path::native(RawFilename, Filename);
158   NotToBeMinimized.insert(Filename);
159 }
160 
161 bool DependencyScanningWorkerFilesystem::shouldMinimize(StringRef RawFilename) {
162   if (!shouldMinimizeBasedOnExtension(RawFilename))
163     return false;
164 
165   llvm::SmallString<256> Filename;
166   llvm::sys::path::native(RawFilename, Filename);
167   return !NotToBeMinimized.contains(Filename);
168 }
169 
170 CachedFileSystemEntry DependencyScanningWorkerFilesystem::createFileSystemEntry(
171     llvm::ErrorOr<llvm::vfs::Status> &&MaybeStatus, StringRef Filename,
172     bool ShouldMinimize) {
173   if (!MaybeStatus)
174     return CachedFileSystemEntry(MaybeStatus.getError());
175 
176   if (MaybeStatus->isDirectory())
177     return CachedFileSystemEntry::createDirectoryEntry(std::move(*MaybeStatus));
178 
179   return CachedFileSystemEntry::createFileEntry(Filename, getUnderlyingFS(),
180                                                 ShouldMinimize);
181 }
182 
183 llvm::ErrorOr<const CachedFileSystemEntry *>
184 DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
185     const StringRef Filename) {
186   bool ShouldMinimize = shouldMinimize(Filename);
187 
188   if (const auto *Entry = Cache.getCachedEntry(Filename, ShouldMinimize))
189     return Entry;
190 
191   // FIXME: Handle PCM/PCH files.
192   // FIXME: Handle module map files.
193 
194   DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
195       &SharedCacheEntry = SharedCache.get(Filename, ShouldMinimize);
196   const CachedFileSystemEntry *Result;
197   {
198     std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
199     CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value;
200 
201     if (!CacheEntry.isValid()) {
202       auto MaybeStatus = getUnderlyingFS().status(Filename);
203       if (!MaybeStatus && !shouldCacheStatFailures(Filename))
204         // HACK: We need to always restat non source files if the stat fails.
205         //   This is because Clang first looks up the module cache and module
206         //   files before building them, and then looks for them again. If we
207         //   cache the stat failure, it won't see them the second time.
208         return MaybeStatus.getError();
209       CacheEntry = createFileSystemEntry(std::move(MaybeStatus), Filename,
210                                          ShouldMinimize);
211     }
212 
213     Result = &CacheEntry;
214   }
215 
216   // Store the result in the local cache.
217   Cache.setCachedEntry(Filename, ShouldMinimize, Result);
218   return Result;
219 }
220 
221 llvm::ErrorOr<llvm::vfs::Status>
222 DependencyScanningWorkerFilesystem::status(const Twine &Path) {
223   SmallString<256> OwnedFilename;
224   StringRef Filename = Path.toStringRef(OwnedFilename);
225   const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
226       getOrCreateFileSystemEntry(Filename);
227   if (!Result)
228     return Result.getError();
229   return (*Result)->getStatus();
230 }
231 
232 namespace {
233 
234 /// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
235 /// this subclass.
236 class MinimizedVFSFile final : public llvm::vfs::File {
237 public:
238   MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
239                    llvm::vfs::Status Stat)
240       : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
241 
242   static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
243   create(const CachedFileSystemEntry *Entry,
244          ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings);
245 
246   llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
247 
248   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
249   getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
250             bool IsVolatile) override {
251     return std::move(Buffer);
252   }
253 
254   std::error_code close() override { return {}; }
255 
256 private:
257   std::unique_ptr<llvm::MemoryBuffer> Buffer;
258   llvm::vfs::Status Stat;
259 };
260 
261 } // end anonymous namespace
262 
263 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MinimizedVFSFile::create(
264     const CachedFileSystemEntry *Entry,
265     ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) {
266   if (Entry->isDirectory())
267     return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
268         std::make_error_code(std::errc::is_a_directory));
269   llvm::ErrorOr<StringRef> Contents = Entry->getContents();
270   if (!Contents)
271     return Contents.getError();
272   auto Result = std::make_unique<MinimizedVFSFile>(
273       llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(),
274                                        /*RequiresNullTerminator=*/false),
275       *Entry->getStatus());
276   if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings)
277     (*PPSkipMappings)[Result->Buffer->getBufferStart()] =
278         &Entry->getPPSkippedRangeMapping();
279   return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
280       std::unique_ptr<llvm::vfs::File>(std::move(Result)));
281 }
282 
283 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
284 DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
285   SmallString<256> OwnedFilename;
286   StringRef Filename = Path.toStringRef(OwnedFilename);
287 
288   const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
289       getOrCreateFileSystemEntry(Filename);
290   if (!Result)
291     return Result.getError();
292   return MinimizedVFSFile::create(Result.get(), PPSkipMappings);
293 }
294