From 0719b3527f5bc550d3677161ca49a0e6d462c9e1 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Fri, 8 Feb 2019 01:20:54 +0000 Subject: [PATCH] [libFuzzer] refactor the way we choose the element to cross-over with, NFC (expected1); add a flag -seed_inputs= to pass extra seed inputs as file paths, not dirs llvm-svn: 353494 --- compiler-rt/lib/fuzzer/FuzzerDriver.cpp | 14 +++++++++++++- compiler-rt/lib/fuzzer/FuzzerFlags.def | 2 ++ compiler-rt/lib/fuzzer/FuzzerInternal.h | 6 ++++-- compiler-rt/lib/fuzzer/FuzzerLoop.cpp | 18 +++++++++++++----- compiler-rt/lib/fuzzer/FuzzerMutate.cpp | 16 ++++++++-------- compiler-rt/lib/fuzzer/FuzzerMutate.h | 6 +++--- compiler-rt/test/fuzzer/cross_over.test | 4 ++++ 7 files changed, 47 insertions(+), 19 deletions(-) diff --git a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp index 0f8389cb3560..2bc895d008c2 100644 --- a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp @@ -732,7 +732,19 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { exit(0); } - F->Loop(*Inputs); + // Parse -seed_inputs=file1,file2,... + Vector ExtraSeedFiles; + if (Flags.seed_inputs) { + std::string s = Flags.seed_inputs; + size_t comma_pos; + while ((comma_pos = s.find_last_of(',')) != std::string::npos) { + ExtraSeedFiles.push_back(s.substr(comma_pos + 1)); + s = s.substr(0, comma_pos); + } + ExtraSeedFiles.push_back(s); + } + + F->Loop(*Inputs, ExtraSeedFiles); if (Flags.verbosity) Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(), diff --git a/compiler-rt/lib/fuzzer/FuzzerFlags.def b/compiler-rt/lib/fuzzer/FuzzerFlags.def index 198e4dd91e69..c762bafa7ae8 100644 --- a/compiler-rt/lib/fuzzer/FuzzerFlags.def +++ b/compiler-rt/lib/fuzzer/FuzzerFlags.def @@ -20,6 +20,8 @@ FUZZER_FLAG_INT(len_control, 100, "Try generating small inputs first, " "then try larger inputs over time. Specifies the rate at which the length " "limit is increased (smaller == faster). If 0, immediately try inputs with " "size up to max_len.") +FUZZER_FLAG_STRING(seed_inputs, "A comma-separated list of input files " + "to use as an additional seed corpus") FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.") FUZZER_FLAG_INT(mutate_depth, 5, "Apply this number of consecutive mutations to each input.") diff --git a/compiler-rt/lib/fuzzer/FuzzerInternal.h b/compiler-rt/lib/fuzzer/FuzzerInternal.h index 9950445bc18f..e934e111488d 100644 --- a/compiler-rt/lib/fuzzer/FuzzerInternal.h +++ b/compiler-rt/lib/fuzzer/FuzzerInternal.h @@ -35,8 +35,10 @@ public: Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, FuzzingOptions Options); ~Fuzzer(); - void Loop(const Vector &CorpusDirs); - void ReadAndExecuteSeedCorpora(const Vector &CorpusDirs); + void Loop(const Vector &CorpusDirs, + const Vector &ExtraSeedFiles); + void ReadAndExecuteSeedCorpora(const Vector &CorpusDirs, + const Vector &ExtraSeedFiles); void MinimizeCrashLoop(const Unit &U); void RereadOutputCorpus(size_t MaxSize); diff --git a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp index fb5aa1f11c19..58fd5c3e97b8 100644 --- a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp @@ -630,6 +630,8 @@ void Fuzzer::MutateAndTestOne() { MD.StartMutationSequence(); auto &II = Corpus.ChooseUnitToMutate(MD.GetRand()); + if (Options.DoCrossOver) + MD.SetCrossOverWith(&Corpus.ChooseUnitToMutate(MD.GetRand()).U); const auto &U = II.U; memcpy(BaseSha1, II.Sha1, sizeof(BaseSha1)); assert(CurrentUnitData); @@ -688,7 +690,9 @@ void Fuzzer::PurgeAllocator() { LastAllocatorPurgeAttemptTime = system_clock::now(); } -void Fuzzer::ReadAndExecuteSeedCorpora(const Vector &CorpusDirs) { +void Fuzzer::ReadAndExecuteSeedCorpora( + const Vector &CorpusDirs, + const Vector &ExtraSeedFiles) { const size_t kMaxSaneLen = 1 << 20; const size_t kMinDefaultLen = 4096; Vector SizedFiles; @@ -702,6 +706,11 @@ void Fuzzer::ReadAndExecuteSeedCorpora(const Vector &CorpusDirs) { Dir.c_str()); LastNumFiles = SizedFiles.size(); } + // Add files from -seed_inputs. + for (auto &File : ExtraSeedFiles) + if (auto Size = FileSize(File)) + SizedFiles.push_back({File, Size}); + for (auto &File : SizedFiles) { MaxSize = Max(File.Size, MaxSize); MinSize = Min(File.Size, MinSize); @@ -761,14 +770,13 @@ void Fuzzer::ReadAndExecuteSeedCorpora(const Vector &CorpusDirs) { } } -void Fuzzer::Loop(const Vector &CorpusDirs) { - ReadAndExecuteSeedCorpora(CorpusDirs); +void Fuzzer::Loop(const Vector &CorpusDirs, + const Vector &ExtraSeedFiles) { + ReadAndExecuteSeedCorpora(CorpusDirs, ExtraSeedFiles); DFT.Clear(); // No need for DFT any more. TPC.SetPrintNewPCs(Options.PrintNewCovPcs); TPC.SetPrintNewFuncs(Options.PrintNewCovFuncs); system_clock::time_point LastCorpusReload = system_clock::now(); - if (Options.DoCrossOver) - MD.SetCorpus(&Corpus); while (true) { auto Now = system_clock::now(); if (duration_cast(Now - LastCorpusReload).count() >= diff --git a/compiler-rt/lib/fuzzer/FuzzerMutate.cpp b/compiler-rt/lib/fuzzer/FuzzerMutate.cpp index 6dc2eccdbb3c..92e469f4c143 100644 --- a/compiler-rt/lib/fuzzer/FuzzerMutate.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerMutate.cpp @@ -8,12 +8,12 @@ // Mutate a test input. //===----------------------------------------------------------------------===// -#include "FuzzerMutate.h" -#include "FuzzerCorpus.h" #include "FuzzerDefs.h" #include "FuzzerExtFunctions.h" #include "FuzzerIO.h" +#include "FuzzerMutate.h" #include "FuzzerOptions.h" +#include "FuzzerTracePC.h" namespace fuzzer { @@ -72,10 +72,10 @@ size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size, size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size, size_t MaxSize) { - if (!Corpus || Corpus->size() < 2 || Size == 0) + if (Size == 0) return 0; - size_t Idx = Rand(Corpus->size()); - const Unit &Other = (*Corpus)[Idx]; + if (!CrossOverWith) return 0; + const Unit &Other = *CrossOverWith; if (Other.empty()) return 0; CustomCrossOverInPlaceHere.resize(MaxSize); @@ -421,9 +421,9 @@ size_t MutationDispatcher::Mutate_ChangeBinaryInteger(uint8_t *Data, size_t MutationDispatcher::Mutate_CrossOver(uint8_t *Data, size_t Size, size_t MaxSize) { if (Size > MaxSize) return 0; - if (!Corpus || Corpus->size() < 2 || Size == 0) return 0; - size_t Idx = Rand(Corpus->size()); - const Unit &O = (*Corpus)[Idx]; + if (Size == 0) return 0; + if (!CrossOverWith) return 0; + const Unit &O = *CrossOverWith; if (O.empty()) return 0; MutateInPlaceHere.resize(MaxSize); auto &U = MutateInPlaceHere; diff --git a/compiler-rt/lib/fuzzer/FuzzerMutate.h b/compiler-rt/lib/fuzzer/FuzzerMutate.h index 33064d7a72b7..6cbce8027624 100644 --- a/compiler-rt/lib/fuzzer/FuzzerMutate.h +++ b/compiler-rt/lib/fuzzer/FuzzerMutate.h @@ -63,7 +63,7 @@ public: /// Change a 1-, 2-, 4-, or 8-byte integer in interesting ways. size_t Mutate_ChangeBinaryInteger(uint8_t *Data, size_t Size, size_t MaxSize); - /// CrossOver Data with some other element of the corpus. + /// CrossOver Data with CrossOverWith. size_t Mutate_CrossOver(uint8_t *Data, size_t Size, size_t MaxSize); /// Applies one of the configured mutations. @@ -88,7 +88,7 @@ public: void PrintRecommendedDictionary(); - void SetCorpus(const InputCorpus *Corpus) { this->Corpus = Corpus; } + void SetCrossOverWith(const Unit *U) { CrossOverWith = U; } Random &GetRand() { return Rand; } @@ -139,7 +139,7 @@ public: DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; size_t CmpDictionaryEntriesDequeIdx = 0; - const InputCorpus *Corpus = nullptr; + const Unit *CrossOverWith = nullptr; Vector MutateInPlaceHere; Vector MutateWithMaskTemp; // CustomCrossOver needs its own buffer as a custom implementation may call diff --git a/compiler-rt/test/fuzzer/cross_over.test b/compiler-rt/test/fuzzer/cross_over.test index 4641cd428b98..058b5eb2c85c 100644 --- a/compiler-rt/test/fuzzer/cross_over.test +++ b/compiler-rt/test/fuzzer/cross_over.test @@ -11,4 +11,8 @@ RUN: mkdir %t-corpus RUN: echo -n ABCDE00000 > %t-corpus/A RUN: echo -n ZZZZZFGHIJ > %t-corpus/B + RUN: not %run %t-CrossOverTest -max_len=10 -seed=1 -runs=10000000 %t-corpus + +# Test the same thing but using -seed_inputs instead of passing the corpus dir. +RUN: not %run %t-CrossOverTest -max_len=10 -seed=1 -runs=10000000 -seed_inputs=%t-corpus/A,%t-corpus/B