1 //===- FuzzerFork.cpp - run fuzzing in separate subprocesses --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // Spawn and orchestrate separate fuzzing processes.
9 //===----------------------------------------------------------------------===//
10 
11 #include "FuzzerCommand.h"
12 #include "FuzzerFork.h"
13 #include "FuzzerIO.h"
14 #include "FuzzerMerge.h"
15 #include "FuzzerSHA1.h"
16 #include "FuzzerTracePC.h"
17 #include "FuzzerUtil.h"
18 
19 #include <atomic>
20 #include <chrono>
21 #include <fstream>
22 #include <memory>
23 #include <mutex>
24 #include <queue>
25 #include <sstream>
26 #include <thread>
27 
28 namespace fuzzer {
29 
30 struct Stats {
31   size_t number_of_executed_units = 0;
32   size_t peak_rss_mb = 0;
33   size_t average_exec_per_sec = 0;
34 };
35 
36 static Stats ParseFinalStatsFromLog(const std::string &LogPath) {
37   std::ifstream In(LogPath);
38   std::string Line;
39   Stats Res;
40   struct {
41     const char *Name;
42     size_t *Var;
43   } NameVarPairs[] = {
44       {"stat::number_of_executed_units:", &Res.number_of_executed_units},
45       {"stat::peak_rss_mb:", &Res.peak_rss_mb},
46       {"stat::average_exec_per_sec:", &Res.average_exec_per_sec},
47       {nullptr, nullptr},
48   };
49   while (std::getline(In, Line, '\n')) {
50     if (Line.find("stat::") != 0) continue;
51     std::istringstream ISS(Line);
52     std::string Name;
53     size_t Val;
54     ISS >> Name >> Val;
55     for (size_t i = 0; NameVarPairs[i].Name; i++)
56       if (Name == NameVarPairs[i].Name)
57         *NameVarPairs[i].Var = Val;
58   }
59   return Res;
60 }
61 
62 struct FuzzJob {
63   // Inputs.
64   Command Cmd;
65   std::string CorpusDir;
66   std::string LogPath;
67   std::string CFPath;
68 
69   // Fuzzing Outputs.
70   int ExitCode;
71 
72   ~FuzzJob() {
73     RemoveFile(CFPath);
74     RemoveFile(LogPath);
75     RmDirRecursive(CorpusDir);
76   }
77 };
78 
79 struct GlobalEnv {
80   Vector<std::string> Args;
81   Vector<std::string> CorpusDirs;
82   std::string MainCorpusDir;
83   std::string TempDir;
84   Set<uint32_t> Features, Cov;
85   Vector<std::string> Files;
86   Random *Rand;
87   std::chrono::system_clock::time_point ProcessStartTime;
88   int Verbosity = 0;
89 
90   size_t NumTimeouts = 0;
91   size_t NumOOMs = 0;
92   size_t NumCrashes = 0;
93 
94 
95   size_t NumRuns = 0;
96 
97   size_t secondsSinceProcessStartUp() const {
98     return std::chrono::duration_cast<std::chrono::seconds>(
99                std::chrono::system_clock::now() - ProcessStartTime)
100         .count();
101   }
102 
103   FuzzJob *CreateNewJob(size_t JobId) {
104     Command Cmd(Args);
105     Cmd.removeFlag("fork");
106     for (auto &C : CorpusDirs) // Remove all corpora from the args.
107       Cmd.removeArgument(C);
108     Cmd.addFlag("reload", "0");  // working in an isolated dir, no reload.
109     Cmd.addFlag("print_final_stats", "1");
110     Cmd.addFlag("print_funcs", "0");  // no need to spend time symbolizing.
111     Cmd.addFlag("max_total_time", std::to_string(std::min((size_t)300, JobId)));
112 
113     auto Job = new FuzzJob;
114     std::string Seeds;
115     if (size_t CorpusSubsetSize =
116             std::min(Files.size(), (size_t)sqrt(Files.size() + 2)))
117       for (size_t i = 0; i < CorpusSubsetSize; i++)
118         Seeds += (Seeds.empty() ? "" : ",") +
119                  Files[Rand->SkewTowardsLast(Files.size())];
120     if (!Seeds.empty())
121       Cmd.addFlag("seed_inputs", Seeds);
122     Job->LogPath = DirPlusFile(TempDir, std::to_string(JobId) + ".log");
123     Job->CorpusDir = DirPlusFile(TempDir, "C" + std::to_string(JobId));
124     Job->CFPath = DirPlusFile(TempDir, std::to_string(JobId) + ".merge");
125 
126 
127     Cmd.addArgument(Job->CorpusDir);
128     RmDirRecursive(Job->CorpusDir);
129     MkDir(Job->CorpusDir);
130 
131     Cmd.setOutputFile(Job->LogPath);
132     Cmd.combineOutAndErr();
133 
134     Job->Cmd = Cmd;
135 
136     if (Verbosity >= 2)
137       Printf("Job %zd/%p Created: %s\n", JobId, Job,
138              Job->Cmd.toString().c_str());
139     // Start from very short runs and gradually increase them.
140     return Job;
141   }
142 
143   void RunOneMergeJob(FuzzJob *Job) {
144     Vector<SizedFile> TempFiles;
145     GetSizedFilesFromDir(Job->CorpusDir, &TempFiles);
146 
147     Vector<std::string> FilesToAdd;
148     Set<uint32_t> NewFeatures, NewCov;
149     CrashResistantMerge(Args, {}, TempFiles, &FilesToAdd, Features,
150                         &NewFeatures, Cov, &NewCov, Job->CFPath, false);
151     for (auto &Path : FilesToAdd) {
152       auto U = FileToVector(Path);
153       auto NewPath = DirPlusFile(MainCorpusDir, Hash(U));
154       WriteToFile(U, NewPath);
155       Files.push_back(NewPath);
156     }
157     Features.insert(NewFeatures.begin(), NewFeatures.end());
158     Cov.insert(NewCov.begin(), NewCov.end());
159     for (auto Idx : NewCov)
160       if (auto *TE = TPC.PCTableEntryByIdx(Idx))
161         if (TPC.PcIsFuncEntry(TE))
162           PrintPC("  NEW_FUNC: %p %F %L\n", "",
163                   TPC.GetNextInstructionPc(TE->PC));
164 
165     auto Stats = ParseFinalStatsFromLog(Job->LogPath);
166     NumRuns += Stats.number_of_executed_units;
167     if (!FilesToAdd.empty() || Job->ExitCode != 0)
168       Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd "
169              "oom/timeout/crash: %zd/%zd/%zd time: %zds\n", NumRuns,
170              Cov.size(), Features.size(), Files.size(),
171              Stats.average_exec_per_sec,
172              NumOOMs, NumTimeouts, NumCrashes, secondsSinceProcessStartUp());
173   }
174 };
175 
176 struct JobQueue {
177   std::queue<FuzzJob *> Qu;
178   std::mutex Mu;
179 
180   void Push(FuzzJob *Job) {
181     std::lock_guard<std::mutex> Lock(Mu);
182     Qu.push(Job);
183   }
184   FuzzJob *Pop() {
185     std::lock_guard<std::mutex> Lock(Mu);
186     if (Qu.empty()) return nullptr;
187     auto Job = Qu.front();
188     Qu.pop();
189     return Job;
190   }
191 };
192 
193 void WorkerThread(std::atomic<bool> *Stop, JobQueue *FuzzQ, JobQueue *MergeQ) {
194   while (!Stop->load()) {
195     auto Job = FuzzQ->Pop();
196     // Printf("WorkerThread: job %p\n", Job);
197     if (!Job) {
198       SleepSeconds(1);
199       continue;
200     }
201     Job->ExitCode = ExecuteCommand(Job->Cmd);
202     MergeQ->Push(Job);
203   }
204 }
205 
206 // This is just a skeleton of an experimental -fork=1 feature.
207 void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
208                   const Vector<std::string> &Args,
209                   const Vector<std::string> &CorpusDirs, int NumJobs) {
210   Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs);
211 
212   GlobalEnv Env;
213   Env.Args = Args;
214   Env.CorpusDirs = CorpusDirs;
215   Env.Rand = &Rand;
216   Env.Verbosity = Options.Verbosity;
217   Env.ProcessStartTime = std::chrono::system_clock::now();
218 
219   Vector<SizedFile> SeedFiles;
220   for (auto &Dir : CorpusDirs)
221     GetSizedFilesFromDir(Dir, &SeedFiles);
222   std::sort(SeedFiles.begin(), SeedFiles.end());
223   Env.TempDir = TempPath(".dir");
224   RmDirRecursive(Env.TempDir);  // in case there is a leftover from old runs.
225   MkDir(Env.TempDir);
226 
227 
228   if (CorpusDirs.empty())
229     MkDir(Env.MainCorpusDir = DirPlusFile(Env.TempDir, "C"));
230   else
231     Env.MainCorpusDir = CorpusDirs[0];
232 
233   auto CFPath = DirPlusFile(Env.TempDir, "merge.txt");
234   CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features,
235                       {}, &Env.Cov,
236                       CFPath, false);
237   RemoveFile(CFPath);
238   Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs,
239          Env.Files.size(), Env.TempDir.c_str());
240 
241   int ExitCode = 0;
242 
243   JobQueue FuzzQ, MergeQ;
244   std::atomic<bool> Stop(false);
245 
246   size_t JobId = 1;
247   Vector<std::thread> Threads;
248   for (int t = 0; t < NumJobs; t++) {
249     Threads.push_back(std::thread(WorkerThread, &Stop, &FuzzQ, &MergeQ));
250     FuzzQ.Push(Env.CreateNewJob(JobId++));
251   }
252 
253   while (true) {
254     std::unique_ptr<FuzzJob> Job(MergeQ.Pop());
255     if (!Job) {
256       if (Stop)
257         break;
258       SleepSeconds(1);
259       continue;
260     }
261     ExitCode = Job->ExitCode;
262     if (ExitCode == Options.InterruptExitCode) {
263       Printf("==%lu== libFuzzer: a child was interrupted; exiting\n", GetPid());
264       Stop = true;
265       break;
266     }
267 
268     Env.RunOneMergeJob(Job.get());
269 
270     // Continue if our crash is one of the ignorred ones.
271     if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode)
272       Env.NumTimeouts++;
273     else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode)
274       Env.NumOOMs++;
275     else if (ExitCode != 0) {
276       Env.NumCrashes++;
277       if (Options.IgnoreCrashes) {
278         std::ifstream In(Job->LogPath);
279         std::string Line;
280         while (std::getline(In, Line, '\n'))
281           if (Line.find("ERROR:") != Line.npos)
282             Printf("%s\n", Line.c_str());
283       } else {
284         // And exit if we don't ignore this crash.
285         Printf("INFO: log from the inner process:\n%s",
286                FileToString(Job->LogPath).c_str());
287         Stop = true;
288       }
289     }
290 
291     // Stop if we are over the time budget.
292     // This is not precise, since other threads are still running
293     // and we will wait while joining them.
294     // We also don't stop instantly: other jobs need to finish.
295     if (Options.MaxTotalTimeSec > 0 && !Stop &&
296         Env.secondsSinceProcessStartUp() >= (size_t)Options.MaxTotalTimeSec) {
297       Printf("INFO: fuzzed for %zd seconds, wrapping up soon\n",
298              Env.secondsSinceProcessStartUp());
299       Stop = true;
300     }
301 
302     if (!Stop)
303       FuzzQ.Push(Env.CreateNewJob(JobId++));
304   }
305   Stop = true;
306 
307   for (auto &T : Threads)
308     T.join();
309 
310   // The workers have terminated. Don't try to remove the directory before they
311   // terminate to avoid a race condition preventing cleanup on Windows.
312   RmDirRecursive(Env.TempDir);
313 
314   // Use the exit code from the last child process.
315   Printf("INFO: exiting: %d time: %zds\n", ExitCode,
316          Env.secondsSinceProcessStartUp());
317   exit(ExitCode);
318 }
319 
320 } // namespace fuzzer
321