1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the pruning of a directory based on least recently used.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/Support/CachePruning.h"
14 
15 #include "llvm/Support/Debug.h"
16 #include "llvm/Support/Errc.h"
17 #include "llvm/Support/Error.h"
18 #include "llvm/Support/FileSystem.h"
19 #include "llvm/Support/Path.h"
20 #include "llvm/Support/raw_ostream.h"
21 
22 #define DEBUG_TYPE "cache-pruning"
23 
24 #include <set>
25 #include <system_error>
26 
27 using namespace llvm;
28 
29 namespace {
30 struct FileInfo {
31   sys::TimePoint<> Time;
32   uint64_t Size;
33   std::string Path;
34 
35   /// Used to determine which files to prune first. Also used to determine
36   /// set membership, so must take into account all fields.
37   bool operator<(const FileInfo &Other) const {
38     if (Time < Other.Time)
39       return true;
40     else if (Other.Time < Time)
41       return false;
42     if (Other.Size < Size)
43       return true;
44     else if (Size < Other.Size)
45       return false;
46     return Path < Other.Path;
47   }
48 };
49 } // anonymous namespace
50 
51 /// Write a new timestamp file with the given path. This is used for the pruning
52 /// interval option.
53 static void writeTimestampFile(StringRef TimestampFile) {
54   std::error_code EC;
55   raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
56 }
57 
58 static Expected<std::chrono::seconds> parseDuration(StringRef Duration) {
59   if (Duration.empty())
60     return make_error<StringError>("Duration must not be empty",
61                                    inconvertibleErrorCode());
62 
63   StringRef NumStr = Duration.slice(0, Duration.size()-1);
64   uint64_t Num;
65   if (NumStr.getAsInteger(0, Num))
66     return make_error<StringError>("'" + NumStr + "' not an integer",
67                                    inconvertibleErrorCode());
68 
69   switch (Duration.back()) {
70   case 's':
71     return std::chrono::seconds(Num);
72   case 'm':
73     return std::chrono::minutes(Num);
74   case 'h':
75     return std::chrono::hours(Num);
76   default:
77     return make_error<StringError>("'" + Duration +
78                                        "' must end with one of 's', 'm' or 'h'",
79                                    inconvertibleErrorCode());
80   }
81 }
82 
83 Expected<CachePruningPolicy>
84 llvm::parseCachePruningPolicy(StringRef PolicyStr) {
85   CachePruningPolicy Policy;
86   std::pair<StringRef, StringRef> P = {"", PolicyStr};
87   while (!P.second.empty()) {
88     P = P.second.split(':');
89 
90     StringRef Key, Value;
91     std::tie(Key, Value) = P.first.split('=');
92     if (Key == "prune_interval") {
93       auto DurationOrErr = parseDuration(Value);
94       if (!DurationOrErr)
95         return DurationOrErr.takeError();
96       Policy.Interval = *DurationOrErr;
97     } else if (Key == "prune_after") {
98       auto DurationOrErr = parseDuration(Value);
99       if (!DurationOrErr)
100         return DurationOrErr.takeError();
101       Policy.Expiration = *DurationOrErr;
102     } else if (Key == "cache_size") {
103       if (Value.back() != '%')
104         return make_error<StringError>("'" + Value + "' must be a percentage",
105                                        inconvertibleErrorCode());
106       StringRef SizeStr = Value.drop_back();
107       uint64_t Size;
108       if (SizeStr.getAsInteger(0, Size))
109         return make_error<StringError>("'" + SizeStr + "' not an integer",
110                                        inconvertibleErrorCode());
111       if (Size > 100)
112         return make_error<StringError>("'" + SizeStr +
113                                            "' must be between 0 and 100",
114                                        inconvertibleErrorCode());
115       Policy.MaxSizePercentageOfAvailableSpace = Size;
116     } else if (Key == "cache_size_bytes") {
117       uint64_t Mult = 1;
118       switch (tolower(Value.back())) {
119       case 'k':
120         Mult = 1024;
121         Value = Value.drop_back();
122         break;
123       case 'm':
124         Mult = 1024 * 1024;
125         Value = Value.drop_back();
126         break;
127       case 'g':
128         Mult = 1024 * 1024 * 1024;
129         Value = Value.drop_back();
130         break;
131       }
132       uint64_t Size;
133       if (Value.getAsInteger(0, Size))
134         return make_error<StringError>("'" + Value + "' not an integer",
135                                        inconvertibleErrorCode());
136       Policy.MaxSizeBytes = Size * Mult;
137     } else if (Key == "cache_size_files") {
138       if (Value.getAsInteger(0, Policy.MaxSizeFiles))
139         return make_error<StringError>("'" + Value + "' not an integer",
140                                        inconvertibleErrorCode());
141     } else {
142       return make_error<StringError>("Unknown key: '" + Key + "'",
143                                      inconvertibleErrorCode());
144     }
145   }
146 
147   return Policy;
148 }
149 
150 /// Prune the cache of files that haven't been accessed in a long time.
151 bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
152   using namespace std::chrono;
153 
154   if (Path.empty())
155     return false;
156 
157   bool isPathDir;
158   if (sys::fs::is_directory(Path, isPathDir))
159     return false;
160 
161   if (!isPathDir)
162     return false;
163 
164   Policy.MaxSizePercentageOfAvailableSpace =
165       std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
166 
167   if (Policy.Expiration == seconds(0) &&
168       Policy.MaxSizePercentageOfAvailableSpace == 0 &&
169       Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
170     LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n");
171     // Nothing will be pruned, early exit
172     return false;
173   }
174 
175   // Try to stat() the timestamp file.
176   SmallString<128> TimestampFile(Path);
177   sys::path::append(TimestampFile, "llvmcache.timestamp");
178   sys::fs::file_status FileStatus;
179   const auto CurrentTime = system_clock::now();
180   if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
181     if (EC == errc::no_such_file_or_directory) {
182       // If the timestamp file wasn't there, create one now.
183       writeTimestampFile(TimestampFile);
184     } else {
185       // Unknown error?
186       return false;
187     }
188   } else {
189     if (!Policy.Interval)
190       return false;
191     if (Policy.Interval != seconds(0)) {
192       // Check whether the time stamp is older than our pruning interval.
193       // If not, do nothing.
194       const auto TimeStampModTime = FileStatus.getLastModificationTime();
195       auto TimeStampAge = CurrentTime - TimeStampModTime;
196       if (TimeStampAge <= *Policy.Interval) {
197         LLVM_DEBUG(dbgs() << "Timestamp file too recent ("
198                           << duration_cast<seconds>(TimeStampAge).count()
199                           << "s old), do not prune.\n");
200         return false;
201       }
202     }
203     // Write a new timestamp file so that nobody else attempts to prune.
204     // There is a benign race condition here, if two processes happen to
205     // notice at the same time that the timestamp is out-of-date.
206     writeTimestampFile(TimestampFile);
207   }
208 
209   // Keep track of files to delete to get below the size limit.
210   // Order by time of last use so that recently used files are preserved.
211   std::set<FileInfo> FileInfos;
212   uint64_t TotalSize = 0;
213 
214   // Walk the entire directory cache, looking for unused files.
215   std::error_code EC;
216   SmallString<128> CachePathNative;
217   sys::path::native(Path, CachePathNative);
218   // Walk all of the files within this directory.
219   for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
220        File != FileEnd && !EC; File.increment(EC)) {
221     // Ignore any files not beginning with the string "llvmcache-". This
222     // includes the timestamp file as well as any files created by the user.
223     // This acts as a safeguard against data loss if the user specifies the
224     // wrong directory as their cache directory.
225     if (!sys::path::filename(File->path()).startswith("llvmcache-"))
226       continue;
227 
228     // Look at this file. If we can't stat it, there's nothing interesting
229     // there.
230     ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
231     if (!StatusOrErr) {
232       LLVM_DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
233       continue;
234     }
235 
236     // If the file hasn't been used recently enough, delete it
237     const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
238     auto FileAge = CurrentTime - FileAccessTime;
239     if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
240       LLVM_DEBUG(dbgs() << "Remove " << File->path() << " ("
241                         << duration_cast<seconds>(FileAge).count()
242                         << "s old)\n");
243       sys::fs::remove(File->path());
244       continue;
245     }
246 
247     // Leave it here for now, but add it to the list of size-based pruning.
248     TotalSize += StatusOrErr->getSize();
249     FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()});
250   }
251 
252   auto FileInfo = FileInfos.begin();
253   size_t NumFiles = FileInfos.size();
254 
255   auto RemoveCacheFile = [&]() {
256     // Remove the file.
257     sys::fs::remove(FileInfo->Path);
258     // Update size
259     TotalSize -= FileInfo->Size;
260     NumFiles--;
261     LLVM_DEBUG(dbgs() << " - Remove " << FileInfo->Path << " (size "
262                       << FileInfo->Size << "), new occupancy is " << TotalSize
263                       << "%\n");
264     ++FileInfo;
265   };
266 
267   // Prune for number of files.
268   if (Policy.MaxSizeFiles)
269     while (NumFiles > Policy.MaxSizeFiles)
270       RemoveCacheFile();
271 
272   // Prune for size now if needed
273   if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
274     auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
275     if (!ErrOrSpaceInfo) {
276       report_fatal_error("Can't get available size");
277     }
278     sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
279     auto AvailableSpace = TotalSize + SpaceInfo.free;
280 
281     if (Policy.MaxSizePercentageOfAvailableSpace == 0)
282       Policy.MaxSizePercentageOfAvailableSpace = 100;
283     if (Policy.MaxSizeBytes == 0)
284       Policy.MaxSizeBytes = AvailableSpace;
285     auto TotalSizeTarget = std::min<uint64_t>(
286         AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
287         Policy.MaxSizeBytes);
288 
289     LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
290                       << "% target is: "
291                       << Policy.MaxSizePercentageOfAvailableSpace << "%, "
292                       << Policy.MaxSizeBytes << " bytes\n");
293 
294     // Remove the oldest accessed files first, till we get below the threshold.
295     while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())
296       RemoveCacheFile();
297   }
298   return true;
299 }
300