1 //===-- FileCollector.cpp ---------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/Support/FileCollector.h"
10 #include "llvm/ADT/SmallString.h"
11 #include "llvm/ADT/Twine.h"
12 #include "llvm/Support/FileSystem.h"
13 #include "llvm/Support/Path.h"
14 #include "llvm/Support/Process.h"
15 
16 using namespace llvm;
17 
18 FileCollectorBase::FileCollectorBase() = default;
19 FileCollectorBase::~FileCollectorBase() = default;
20 
21 void FileCollectorBase::addFile(const Twine &File) {
22   std::lock_guard<std::mutex> lock(Mutex);
23   std::string FileStr = File.str();
24   if (markAsSeen(FileStr))
25     addFileImpl(FileStr);
26 }
27 
28 void FileCollectorBase::addDirectory(const Twine &Dir) {
29   assert(sys::fs::is_directory(Dir));
30   std::error_code EC;
31   addDirectoryImpl(Dir, vfs::getRealFileSystem(), EC);
32 }
33 
34 static bool isCaseSensitivePath(StringRef Path) {
35   SmallString<256> TmpDest = Path, UpperDest, RealDest;
36 
37   // Remove component traversals, links, etc.
38   if (!sys::fs::real_path(Path, TmpDest))
39     return true; // Current default value in vfs.yaml
40   Path = TmpDest;
41 
42   // Change path to all upper case and ask for its real path, if the latter
43   // exists and is equal to path, it's not case sensitive. Default to case
44   // sensitive in the absence of real_path, since this is the YAMLVFSWriter
45   // default.
46   UpperDest = Path.upper();
47   if (sys::fs::real_path(UpperDest, RealDest) && Path.equals(RealDest))
48     return false;
49   return true;
50 }
51 
52 FileCollector::FileCollector(std::string Root, std::string OverlayRoot)
53     : Root(std::move(Root)), OverlayRoot(std::move(OverlayRoot)) {
54 }
55 
56 bool FileCollector::getRealPath(StringRef SrcPath,
57                                 SmallVectorImpl<char> &Result) {
58   SmallString<256> RealPath;
59   StringRef FileName = sys::path::filename(SrcPath);
60   std::string Directory = sys::path::parent_path(SrcPath).str();
61   auto DirWithSymlink = SymlinkMap.find(Directory);
62 
63   // Use real_path to fix any symbolic link component present in a path.
64   // Computing the real path is expensive, cache the search through the parent
65   // path Directory.
66   if (DirWithSymlink == SymlinkMap.end()) {
67     auto EC = sys::fs::real_path(Directory, RealPath);
68     if (EC)
69       return false;
70     SymlinkMap[Directory] = std::string(RealPath.str());
71   } else {
72     RealPath = DirWithSymlink->second;
73   }
74 
75   sys::path::append(RealPath, FileName);
76   Result.swap(RealPath);
77   return true;
78 }
79 
80 void FileCollector::addFileImpl(StringRef SrcPath) {
81   // We need an absolute src path to append to the root.
82   SmallString<256> AbsoluteSrc = SrcPath;
83   sys::fs::make_absolute(AbsoluteSrc);
84 
85   // Canonicalize src to a native path to avoid mixed separator styles.
86   sys::path::native(AbsoluteSrc);
87 
88   // Remove redundant leading "./" pieces and consecutive separators.
89   AbsoluteSrc = sys::path::remove_leading_dotslash(AbsoluteSrc);
90 
91   // Canonicalize the source path by removing "..", "." components.
92   SmallString<256> VirtualPath = AbsoluteSrc;
93   sys::path::remove_dots(VirtualPath, /*remove_dot_dot=*/true);
94 
95   // If a ".." component is present after a symlink component, remove_dots may
96   // lead to the wrong real destination path. Let the source be canonicalized
97   // like that but make sure we always use the real path for the destination.
98   SmallString<256> CopyFrom;
99   if (!getRealPath(AbsoluteSrc, CopyFrom))
100     CopyFrom = VirtualPath;
101 
102   SmallString<256> DstPath = StringRef(Root);
103   sys::path::append(DstPath, sys::path::relative_path(CopyFrom));
104 
105   // Always map a canonical src path to its real path into the YAML, by doing
106   // this we map different virtual src paths to the same entry in the VFS
107   // overlay, which is a way to emulate symlink inside the VFS; this is also
108   // needed for correctness, not doing that can lead to module redefinition
109   // errors.
110   addFileToMapping(VirtualPath, DstPath);
111 }
112 
113 llvm::vfs::directory_iterator
114 FileCollector::addDirectoryImpl(const llvm::Twine &Dir,
115                                 IntrusiveRefCntPtr<vfs::FileSystem> FS,
116                                 std::error_code &EC) {
117   auto It = FS->dir_begin(Dir, EC);
118   if (EC)
119     return It;
120   addFile(Dir);
121   for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
122     if (It->type() == sys::fs::file_type::regular_file ||
123         It->type() == sys::fs::file_type::directory_file ||
124         It->type() == sys::fs::file_type::symlink_file) {
125       addFile(It->path());
126     }
127   }
128   if (EC)
129     return It;
130   // Return a new iterator.
131   return FS->dir_begin(Dir, EC);
132 }
133 
134 /// Set the access and modification time for the given file from the given
135 /// status object.
136 static std::error_code
137 copyAccessAndModificationTime(StringRef Filename,
138                               const sys::fs::file_status &Stat) {
139   int FD;
140 
141   if (auto EC =
142           sys::fs::openFileForWrite(Filename, FD, sys::fs::CD_OpenExisting))
143     return EC;
144 
145   if (auto EC = sys::fs::setLastAccessAndModificationTime(
146           FD, Stat.getLastAccessedTime(), Stat.getLastModificationTime()))
147     return EC;
148 
149   if (auto EC = sys::Process::SafelyCloseFileDescriptor(FD))
150     return EC;
151 
152   return {};
153 }
154 
155 std::error_code FileCollector::copyFiles(bool StopOnError) {
156   auto Err = sys::fs::create_directories(Root, /*IgnoreExisting=*/true);
157   if (Err) {
158     return Err;
159   }
160 
161   std::lock_guard<std::mutex> lock(Mutex);
162 
163   for (auto &entry : VFSWriter.getMappings()) {
164     // Get the status of the original file/directory.
165     sys::fs::file_status Stat;
166     if (std::error_code EC = sys::fs::status(entry.VPath, Stat)) {
167       if (StopOnError)
168         return EC;
169       continue;
170     }
171 
172     // Continue if the file doesn't exist.
173     if (Stat.type() == sys::fs::file_type::file_not_found)
174       continue;
175 
176     // Create directory tree.
177     if (std::error_code EC =
178             sys::fs::create_directories(sys::path::parent_path(entry.RPath),
179                                         /*IgnoreExisting=*/true)) {
180       if (StopOnError)
181         return EC;
182     }
183 
184     if (Stat.type() == sys::fs::file_type::directory_file) {
185       // Construct a directory when it's just a directory entry.
186       if (std::error_code EC =
187               sys::fs::create_directories(entry.RPath,
188                                           /*IgnoreExisting=*/true)) {
189         if (StopOnError)
190           return EC;
191       }
192       continue;
193     }
194 
195     // Copy file over.
196     if (std::error_code EC = sys::fs::copy_file(entry.VPath, entry.RPath)) {
197       if (StopOnError)
198         return EC;
199     }
200 
201     // Copy over permissions.
202     if (auto perms = sys::fs::getPermissions(entry.VPath)) {
203       if (std::error_code EC = sys::fs::setPermissions(entry.RPath, *perms)) {
204         if (StopOnError)
205           return EC;
206       }
207     }
208 
209     // Copy over modification time.
210     copyAccessAndModificationTime(entry.RPath, Stat);
211   }
212   return {};
213 }
214 
215 std::error_code FileCollector::writeMapping(StringRef MappingFile) {
216   std::lock_guard<std::mutex> lock(Mutex);
217 
218   VFSWriter.setOverlayDir(OverlayRoot);
219   VFSWriter.setCaseSensitivity(isCaseSensitivePath(OverlayRoot));
220   VFSWriter.setUseExternalNames(false);
221 
222   std::error_code EC;
223   raw_fd_ostream os(MappingFile, EC, sys::fs::OF_Text);
224   if (EC)
225     return EC;
226 
227   VFSWriter.write(os);
228 
229   return {};
230 }
231 
232 namespace llvm {
233 
234 class FileCollectorFileSystem : public vfs::FileSystem {
235 public:
236   explicit FileCollectorFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS,
237                                    std::shared_ptr<FileCollector> Collector)
238       : FS(std::move(FS)), Collector(std::move(Collector)) {}
239 
240   llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override {
241     auto Result = FS->status(Path);
242     if (Result && Result->exists())
243       Collector->addFile(Path);
244     return Result;
245   }
246 
247   llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
248   openFileForRead(const Twine &Path) override {
249     auto Result = FS->openFileForRead(Path);
250     if (Result && *Result)
251       Collector->addFile(Path);
252     return Result;
253   }
254 
255   llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,
256                                           std::error_code &EC) override {
257     return Collector->addDirectoryImpl(Dir, FS, EC);
258   }
259 
260   std::error_code getRealPath(const Twine &Path,
261                               SmallVectorImpl<char> &Output) const override {
262     auto EC = FS->getRealPath(Path, Output);
263     if (!EC) {
264       Collector->addFile(Path);
265       if (Output.size() > 0)
266         Collector->addFile(Output);
267     }
268     return EC;
269   }
270 
271   std::error_code isLocal(const Twine &Path, bool &Result) override {
272     return FS->isLocal(Path, Result);
273   }
274 
275   llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
276     return FS->getCurrentWorkingDirectory();
277   }
278 
279   std::error_code setCurrentWorkingDirectory(const llvm::Twine &Path) override {
280     return FS->setCurrentWorkingDirectory(Path);
281   }
282 
283 private:
284   IntrusiveRefCntPtr<vfs::FileSystem> FS;
285   std::shared_ptr<FileCollector> Collector;
286 };
287 
288 } // namespace llvm
289 
290 IntrusiveRefCntPtr<vfs::FileSystem>
291 FileCollector::createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,
292                                   std::shared_ptr<FileCollector> Collector) {
293   return new FileCollectorFileSystem(std::move(BaseFS), std::move(Collector));
294 }
295