diff --git a/centipede/BUILD b/centipede/BUILD index db3134503..fdbaf2f39 100644 --- a/centipede/BUILD +++ b/centipede/BUILD @@ -366,7 +366,7 @@ cc_library( deps = [ ":centipede_callbacks", ":environment", - ":mutation_input", + ":mutation_data", ":runner_result", ":stop", ":thread_pool", @@ -439,6 +439,7 @@ cc_library( # used in centipede_runner. ":feature", ":execution_metadata", + ":mutation_data", ":shared_memory_blob_sequence", "@com_google_fuzztest//common:defs", ], @@ -454,14 +455,14 @@ cc_library( # used in centipede_runner. ":shared_memory_blob_sequence", ":execution_metadata", - ":mutation_input", + ":mutation_data", "@com_google_fuzztest//common:defs", ], ) cc_library( - name = "mutation_input", - hdrs = ["mutation_input.h"], + name = "mutation_data", + hdrs = ["mutation_data.h"], copts = DISABLE_SANCOV_COPTS, deps = [ # This target must have a minimal set of dependencies since it is @@ -480,7 +481,7 @@ cc_library( deps = [ ":execution_metadata", ":knobs", - ":mutation_input", + ":mutation_data", "@abseil-cpp//absl/base:nullability", "@com_google_fuzztest//common:defs", ], @@ -624,7 +625,7 @@ cc_library( ":control_flow", ":environment", ":fuzztest_mutator", - ":mutation_input", + ":mutation_data", ":runner_request", ":runner_result", ":shared_memory_blob_sequence", @@ -716,7 +717,7 @@ cc_library( ":environment", ":feature", ":feature_set", - ":mutation_input", + ":mutation_data", ":pc_info", ":runner_result", ":rusage_profiler", @@ -852,7 +853,7 @@ cc_library( deps = [ ":centipede_callbacks", ":environment", - ":mutation_input", + ":mutation_data", ":runner_result", ":stop", "@abseil-cpp//absl/status", @@ -870,7 +871,7 @@ cc_library( ":byte_array_mutator", ":execution_metadata", ":knobs", - ":mutation_input", + ":mutation_data", "@abseil-cpp//absl/random", "@abseil-cpp//absl/types:span", "@com_google_fuzztest//common:defs", @@ -938,6 +939,7 @@ cc_library( name = "runner_cmp_trace", hdrs = ["runner_cmp_trace.h"], copts = DISABLE_SANCOV_COPTS, + deps = ["@abseil-cpp//absl/base:core_headers"], ) # Library for manipulating centipede runner flags. This is not used by the @@ -1030,7 +1032,7 @@ RUNNER_DEPS = [ ":foreach_nonzero", ":int_utils", ":knobs", - ":mutation_input", + ":mutation_data", ":rolling_hash", ":runner_cmp_trace", ":runner_fork_server", @@ -1095,7 +1097,7 @@ cc_library( linkstatic = True, # Must be linked statically even when dynamic_mode=on. deps = [ ":centipede_runner_no_main", - ":mutation_input", + ":mutation_data", "@abseil-cpp//absl/base:nullability", "@com_google_fuzztest//common:defs", ], @@ -1234,7 +1236,7 @@ cc_library( ":corpus", ":environment", ":feature", - ":mutation_input", + ":mutation_data", ":runner_result", ":util", "@com_google_fuzztest//common:defs", @@ -1506,6 +1508,7 @@ cc_test( deps = [ ":execution_metadata", ":feature", + ":mutation_data", ":runner_result", ":shared_memory_blob_sequence", "@com_google_fuzztest//common:defs", @@ -1515,10 +1518,10 @@ cc_test( ) cc_test( - name = "mutation_input_test", - srcs = ["mutation_input_test.cc"], + name = "mutation_data_test", + srcs = ["mutation_data_test.cc"], deps = [ - ":mutation_input", + ":mutation_data", "@com_google_fuzztest//common:defs", "@googletest//:gtest_main", ], @@ -1531,7 +1534,7 @@ cc_test( ":byte_array_mutator", ":execution_metadata", ":knobs", - ":mutation_input", + ":mutation_data", ":runner_cmp_trace", "@abseil-cpp//absl/container:flat_hash_set", "@com_google_fuzztest//common:defs", @@ -1564,7 +1567,7 @@ cc_test( ":execution_metadata", ":fuzztest_mutator", ":knobs", - ":mutation_input", + ":mutation_data", "@abseil-cpp//absl/container:flat_hash_set", "@abseil-cpp//absl/strings", "@com_google_fuzztest//common:defs", @@ -1911,7 +1914,7 @@ cc_test( ":centipede_interface", ":environment", ":feature", - ":mutation_input", + ":mutation_data", ":runner_result", ":stop", ":util", diff --git a/centipede/byte_array_mutator.cc b/centipede/byte_array_mutator.cc index 9a29a432e..60834c791 100644 --- a/centipede/byte_array_mutator.cc +++ b/centipede/byte_array_mutator.cc @@ -24,7 +24,7 @@ #include "./centipede/execution_metadata.h" #include "./centipede/knobs.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./common/defs.h" namespace fuzztest::internal { @@ -321,27 +321,28 @@ void ByteArrayMutator::CrossOver(ByteArray &data, const ByteArray &other) { // TODO(kcc): add tests with different values of knobs. const KnobId knob_mutate_or_crossover = Knobs::NewId("mutate_or_crossover"); -std::vector ByteArrayMutator::MutateMany( - const std::vector &inputs, size_t num_mutants) { +std::vector ByteArrayMutator::MutateMany( + const std::vector& inputs, size_t num_mutants) { if (inputs.empty()) abort(); // TODO(xinhaoyuan): Consider metadata in other inputs instead of always the // first one. SetMetadata(inputs[0].metadata != nullptr ? *inputs[0].metadata : ExecutionMetadata()); size_t num_inputs = inputs.size(); - std::vector mutants; + std::vector mutants; mutants.reserve(num_mutants); for (size_t i = 0; i < num_mutants; ++i) { - auto mutant = inputs[rng_() % num_inputs].data; - if (mutant.size() <= max_len_ && + Mutant mutant; + mutant.data = inputs[rng_() % num_inputs].data; + if (mutant.data.size() <= max_len_ && knobs_.GenerateBool(knob_mutate_or_crossover, rng_())) { // Do crossover only if the mutant is not over the max_len_. // Perform crossover with some other input. It may be the same input. const auto &other_input = inputs[rng_() % num_inputs].data; - CrossOver(mutant, other_input); + CrossOver(mutant.data, other_input); } else { // Perform mutation. - Mutate(mutant); + Mutate(mutant.data); } mutants.push_back(std::move(mutant)); } diff --git a/centipede/byte_array_mutator.h b/centipede/byte_array_mutator.h index 3c6978caa..782b2842b 100644 --- a/centipede/byte_array_mutator.h +++ b/centipede/byte_array_mutator.h @@ -25,7 +25,7 @@ #include "absl/base/nullability.h" #include "./centipede/execution_metadata.h" #include "./centipede/knobs.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./common/defs.h" namespace fuzztest::internal { @@ -33,7 +33,7 @@ namespace fuzztest::internal { // A simple class representing an array of up to kMaxEntrySize bytes. class DictEntry { public: - static constexpr uint8_t kMaxEntrySize = 16; + static constexpr uint8_t kMaxEntrySize = 128; explicit DictEntry(ByteSpan bytes) : bytes_{}, // initialize bytes_ to all zeros @@ -108,8 +108,8 @@ class ByteArrayMutator { } // Takes non-empty `inputs` and produces `num_mutants` mutants. - std::vector MutateMany(const std::vector &inputs, - size_t num_mutants); + std::vector MutateMany(const std::vector& inputs, + size_t num_mutants); using CrossOverFn = void (ByteArrayMutator::*)(ByteArray &, const ByteArray &); diff --git a/centipede/byte_array_mutator_test.cc b/centipede/byte_array_mutator_test.cc index ae35641b5..e28dff356 100644 --- a/centipede/byte_array_mutator_test.cc +++ b/centipede/byte_array_mutator_test.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "gmock/gmock.h" @@ -25,7 +26,7 @@ #include "absl/container/flat_hash_set.h" #include "./centipede/execution_metadata.h" #include "./centipede/knobs.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/runner_cmp_trace.h" #include "./common/defs.h" @@ -93,8 +94,9 @@ TEST(ByteArrayMutator, RoundDownToRemoveCorrectly) { namespace { TEST(DictEntry, DictEntry) { - uint8_t bytes[17] = {0, 1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 16}; + uint8_t bytes[129]; + std::iota(bytes, bytes + 129, 0); + DictEntry a_0_10({bytes + 0, 10}); DictEntry a_0_4({bytes + 0, 4}); DictEntry a_1_8({bytes + 1, 8}); @@ -103,7 +105,7 @@ TEST(DictEntry, DictEntry) { EXPECT_LT(a_0_10, a_1_8); EXPECT_EQ(memcmp(a_0_10.begin(), bytes, a_0_10.end() - a_0_10.begin()), 0); - EXPECT_DEATH({ DictEntry a_0_10({bytes, 17}); }, ""); + EXPECT_DEATH({ DictEntry a_0_10({bytes, 129}); }, ""); } TEST(CmpDictionary, CmpDictionary) { @@ -158,11 +160,11 @@ TEST(CmpDictionary, CmpDictionary) { } TEST(CmpDictionary, CmpDictionaryIsCompatibleWithCmpTrace) { - CmpTrace<0, 13> traceN; + CmpTrace<0, 13> traceN = {}; traceN.Clear(); constexpr uint8_t long_array[20] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}; - traceN.Capture(20, long_array, long_array); // will be trimmed to 16. + traceN.Capture(20, long_array, long_array); ExecutionMetadata metadata; bool append_failed = false; @@ -928,12 +930,12 @@ TEST(ByteArrayMutator, MutateManyWithAlignedInputs) { {0, 1, 2, 3, 4, 5, 6, 7}, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }; - const std::vector mutants = + const std::vector mutants = mutator.MutateMany(GetMutationInputRefsFromDataInputs(aligned_inputs), kNumMutantsToGenerate); EXPECT_EQ(mutants.size(), kNumMutantsToGenerate); - for (const ByteArray &mutant : mutants) { - EXPECT_EQ(mutant.size() % kSizeAlignment, 0); + for (const Mutant& mutant : mutants) { + EXPECT_EQ(mutant.data.size() % kSizeAlignment, 0); } } @@ -958,13 +960,13 @@ TEST(ByteArrayMutator, MutateManyWithUnalignedInputs) { {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, }; - const std::vector mutants = + const std::vector mutants = mutator.MutateMany(GetMutationInputRefsFromDataInputs(unaligned_inputs), kNumMutantsToGenerate); EXPECT_EQ(mutants.size(), kNumMutantsToGenerate); - for (const ByteArray &mutant : mutants) { - if (mutant.size() % kSizeAlignment != 0) { - EXPECT_LE(mutant.size(), 11); + for (const Mutant& mutant : mutants) { + if (mutant.data.size() % kSizeAlignment != 0) { + EXPECT_LE(mutant.data.size(), 11); } } } @@ -982,12 +984,12 @@ TEST(ByteArrayMutator, MutateManyWithMaxLen) { {0, 1, 2}, {0, 1, 2, 3}, }; - const std::vector mutants = mutator.MutateMany( + const std::vector mutants = mutator.MutateMany( GetMutationInputRefsFromDataInputs(inputs), kNumMutantsToGenerate); EXPECT_EQ(mutants.size(), kNumMutantsToGenerate); - for (const ByteArray &mutant : mutants) { - EXPECT_LE(mutant.size(), kMaxLen); + for (const Mutant& mutant : mutants) { + EXPECT_LE(mutant.data.size(), kMaxLen); } } @@ -1001,16 +1003,16 @@ TEST(ByteArrayMutator, MutateManyWithMaxLenWithStartingLargeInput) { const std::vector large_input = { {0, 1, 2, 3, 4, 5, 6, 7}, {0}, {0, 1}, {0, 1, 2}, {0, 1, 2, 3}, }; - const std::vector mutants = mutator.MutateMany( + const std::vector mutants = mutator.MutateMany( GetMutationInputRefsFromDataInputs(large_input), kNumMutantsToGenerate); EXPECT_EQ(mutants.size(), kNumMutantsToGenerate); - for (const ByteArray &mutant : mutants) { - if (mutant.size() > kMaxLen) { + for (const Mutant& mutant : mutants) { + if (mutant.data.size() > kMaxLen) { // The only mutant larger than max length should be the same large input // that mutation originally started with. All other mutants should be // within the maximum length specified. - EXPECT_EQ(mutant, large_input[0]); + EXPECT_EQ(mutant.data, large_input[0]); } } } diff --git a/centipede/centipede.cc b/centipede/centipede.cc index b94f24610..1ddcf9790 100644 --- a/centipede/centipede.cc +++ b/centipede/centipede.cc @@ -82,7 +82,7 @@ #include "./centipede/environment.h" #include "./centipede/feature.h" #include "./centipede/feature_set.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/runner_result.h" #include "./centipede/rusage_profiler.h" #include "./centipede/rusage_stats.h" @@ -425,10 +425,10 @@ size_t Centipede::AddPcPairFeatures(FeatureVec &fv) { } bool Centipede::RunBatch( - const std::vector &input_vec, - BlobFileWriter *absl_nullable corpus_file, - BlobFileWriter *absl_nullable features_file, - BlobFileWriter *absl_nullable unconditional_features_file) { + const std::vector& input_vec, + BlobFileWriter* absl_nullable corpus_file, + BlobFileWriter* absl_nullable features_file, + BlobFileWriter* absl_nullable unconditional_features_file) { BatchResult batch_result; bool success = ExecuteAndReportCrash(env_.binary, input_vec, batch_result); FUZZTEST_CHECK_EQ(input_vec.size(), batch_result.results().size()); @@ -866,13 +866,18 @@ void Centipede::FuzzingLoop() { MutationInputRef{corpus_record.data, &corpus_record.metadata}); } - const std::vector mutants = + std::vector mutants = user_callbacks_.Mutate(mutation_inputs, batch_size); if (ShouldStop()) break; + new_runs += mutants.size(); + std::vector inputs; + inputs.reserve(mutants.size()); + for (auto& mutant : mutants) { + inputs.push_back(std::move(mutant.data)); + } bool gained_new_coverage = - RunBatch(mutants, corpus_file.get(), features_file.get(), nullptr); - new_runs += mutants.size(); + RunBatch(inputs, corpus_file.get(), features_file.get(), nullptr); if (gained_new_coverage) { UpdateAndMaybeLogStats("new-feature", 1); diff --git a/centipede/centipede.h b/centipede/centipede.h index 625938f3c..aa25a423e 100644 --- a/centipede/centipede.h +++ b/centipede/centipede.h @@ -84,10 +84,10 @@ class Centipede { // * its features are written to `features_file` (if that's non-null). // Returns true if new features were observed. // Post-condition: `batch_result.results.size()` == `input_vec.size()`. - bool RunBatch(const std::vector &input_vec, - BlobFileWriter *absl_nullable corpus_file, - BlobFileWriter *absl_nullable features_file, - BlobFileWriter *absl_nullable unconditional_features_file); + bool RunBatch(const std::vector& input_vec, + BlobFileWriter* absl_nullable corpus_file, + BlobFileWriter* absl_nullable features_file, + BlobFileWriter* absl_nullable unconditional_features_file); // Loads seed inputs from the user callbacks, execute them, and store them // with the corresponding features into `corpus_file` and `features_file`. void LoadSeedInputs(BlobFileWriter *absl_nonnull corpus_file, diff --git a/centipede/centipede_callbacks.cc b/centipede/centipede_callbacks.cc index 3f6f817dc..51d796ade 100644 --- a/centipede/centipede_callbacks.cc +++ b/centipede/centipede_callbacks.cc @@ -48,7 +48,7 @@ #include "./centipede/binary_info.h" #include "./centipede/command.h" #include "./centipede/control_flow.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/runner_request.h" #include "./centipede/runner_result.h" #include "./centipede/stop.h" diff --git a/centipede/centipede_callbacks.h b/centipede/centipede_callbacks.h index 068dfc6c4..7980e2579 100644 --- a/centipede/centipede_callbacks.h +++ b/centipede/centipede_callbacks.h @@ -30,7 +30,7 @@ #include "./centipede/command.h" #include "./centipede/environment.h" #include "./centipede/fuzztest_mutator.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/runner_result.h" #include "./centipede/shared_memory_blob_sequence.h" #include "./centipede/util.h" @@ -72,8 +72,8 @@ class CentipedeCallbacks { BatchResult &batch_result) = 0; // Takes non-empty `inputs` and returns at most `num_mutants` mutated inputs. - virtual std::vector Mutate( - const std::vector &inputs, size_t num_mutants) { + virtual std::vector Mutate( + const std::vector& inputs, size_t num_mutants) { return env_.use_legacy_default_mutator ? byte_array_mutator_.MutateMany(inputs, num_mutants) : fuzztest_mutator_.MutateMany(inputs, num_mutants); diff --git a/centipede/centipede_default_callbacks.cc b/centipede/centipede_default_callbacks.cc index 73b8b8e41..67fe09fa4 100644 --- a/centipede/centipede_default_callbacks.cc +++ b/centipede/centipede_default_callbacks.cc @@ -25,7 +25,7 @@ #include "absl/status/statusor.h" #include "./centipede/centipede_callbacks.h" #include "./centipede/environment.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/runner_result.h" #include "./centipede/stop.h" #include "./common/defs.h" @@ -72,8 +72,8 @@ CentipedeDefaultCallbacks::GetSerializedTargetConfig() { "Failed to get serialized configuration from the target binary."); } -std::vector CentipedeDefaultCallbacks::Mutate( - const std::vector &inputs, size_t num_mutants) { +std::vector CentipedeDefaultCallbacks::Mutate( + const std::vector& inputs, size_t num_mutants) { if (num_mutants == 0) return {}; // In persistent mode, mutation could fail due to previous asynchronous // failure, thus give it one more chance to mutate in a clean state. diff --git a/centipede/centipede_default_callbacks.h b/centipede/centipede_default_callbacks.h index 0b7856261..9fb157f9e 100644 --- a/centipede/centipede_default_callbacks.h +++ b/centipede/centipede_default_callbacks.h @@ -28,7 +28,7 @@ #include "absl/status/statusor.h" #include "./centipede/centipede_callbacks.h" #include "./centipede/environment.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/runner_result.h" #include "./common/defs.h" @@ -42,8 +42,8 @@ class CentipedeDefaultCallbacks : public CentipedeCallbacks { absl::StatusOr GetSerializedTargetConfig() override; bool Execute(std::string_view binary, const std::vector &inputs, BatchResult &batch_result) override; - std::vector Mutate(const std::vector &inputs, - size_t num_mutants) override; + std::vector Mutate(const std::vector& inputs, + size_t num_mutants) override; private: std::optional custom_mutator_is_usable_ = std::nullopt; diff --git a/centipede/centipede_test.cc b/centipede/centipede_test.cc index 16e928fd1..9454d34bf 100644 --- a/centipede/centipede_test.cc +++ b/centipede/centipede_test.cc @@ -38,7 +38,7 @@ #include "./centipede/centipede_interface.h" #include "./centipede/environment.h" #include "./centipede/feature.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/runner_result.h" #include "./centipede/stop.h" #include "./centipede/util.h" @@ -106,19 +106,19 @@ class CentipedeMock : public CentipedeCallbacks { // (the value {0} is produced by the default GetSeeds()). // Next 65536 mutations are 2-byte sequences {0,0} ... {255, 255}. // Then repeat 2-byte sequences. - std::vector Mutate(const std::vector &inputs, - size_t num_mutants) override { - std::vector mutants; + std::vector Mutate(const std::vector& inputs, + size_t num_mutants) override { + std::vector mutants; mutants.reserve(num_mutants); for (size_t i = 0; i < num_mutants; ++i) { num_mutations_++; if (num_mutations_ < 256) { - mutants.push_back({static_cast(num_mutations_)}); + mutants.push_back({/*data=*/{static_cast(num_mutations_)}}); continue; } uint8_t byte0 = (num_mutations_ - 256) / 256; uint8_t byte1 = (num_mutations_ - 256) % 256; - mutants.push_back({byte0, byte1}); + mutants.push_back({/*data=*/{byte0, byte1}}); } return mutants; } @@ -350,8 +350,8 @@ class MutateCallbacks : public CentipedeCallbacks { } // Will not be called. - std::vector Mutate(const std::vector &inputs, - size_t num_mutants) override { + std::vector Mutate(const std::vector& inputs, + size_t num_mutants) override { FUZZTEST_LOG(FATAL); } @@ -436,8 +436,9 @@ TEST_F(CentipedeWithTemporaryLocalDir, MutateViaExternalBinary) { GetMutationInputRefsFromDataInputs(inputs), 10000); EXPECT_EQ(result.exit_code(), EXIT_SUCCESS); EXPECT_TRUE(result.has_custom_mutator()); - EXPECT_THAT(result.mutants(), AllOf(IsSupersetOf(all_expected_mutants), - Each(Not(IsEmpty())))); + EXPECT_THAT( + GetDataFromMutants(result.mutants()), + AllOf(IsSupersetOf(all_expected_mutants), Each(Not(IsEmpty())))); } } @@ -451,9 +452,10 @@ TEST_F(CentipedeWithTemporaryLocalDir, MutateViaExternalBinary) { 10000); EXPECT_EQ(result.exit_code(), EXIT_SUCCESS); EXPECT_TRUE(result.has_custom_mutator()); - EXPECT_THAT(result.mutants(), AllOf(IsSupersetOf(all_expected_mutants), - Each(Not(IsEmpty())))); - EXPECT_THAT(result.mutants(), + const auto mutant_data = GetDataFromMutants(result.mutants()); + EXPECT_THAT(mutant_data, AllOf(IsSupersetOf(all_expected_mutants), + Each(Not(IsEmpty())))); + EXPECT_THAT(mutant_data, AllOf(IsSupersetOf(all_expected_mutants), Each(Not(IsEmpty())), // The byte_array_mutator may insert up to 20 bytes to an // input, which may push the size over the max_len. @@ -471,9 +473,10 @@ TEST_F(CentipedeWithTemporaryLocalDir, MutateViaExternalBinary) { binary_with_custom_mutator, GetMutationInputRefsFromDataInputs(inputs), 10000); // Must contain normal mutants, but not the ones from crossover. - EXPECT_THAT(result.mutants(), IsSupersetOf(some_of_expected_mutants)); + const auto mutant_data = GetDataFromMutants(result.mutants()); + EXPECT_THAT(mutant_data, IsSupersetOf(some_of_expected_mutants)); for (const auto &crossover_mutant : expected_crossover_mutants) { - EXPECT_THAT(result.mutants(), Not(Contains(crossover_mutant))); + EXPECT_THAT(mutant_data, Not(Contains(crossover_mutant))); } } } @@ -497,12 +500,12 @@ class MergeMock : public CentipedeCallbacks { } // Every consecutive mutation is {number_of_mutations_} (starting from 1). - std::vector Mutate(const std::vector &inputs, - size_t num_mutants) override { - std::vector mutants{num_mutants}; + std::vector Mutate(const std::vector& inputs, + size_t num_mutants) override { + std::vector mutants(num_mutants); for (auto &mutant : mutants) { - mutant.resize(1); - mutant[0] = ++number_of_mutations_; + mutant.data.resize(1); + mutant.data[0] = ++number_of_mutations_; } return mutants; } @@ -582,17 +585,17 @@ class FunctionFilterMock : public CentipedeCallbacks { } // Sets the inputs to one of 3 pre-defined values. - std::vector Mutate(const std::vector &inputs, - size_t num_mutants) override { + std::vector Mutate(const std::vector& inputs, + size_t num_mutants) override { for (auto &input : inputs) { if (!seed_inputs_.contains(input.data)) { observed_inputs_.insert(input.data); } } - std::vector mutants; + std::vector mutants; mutants.reserve(num_mutants); for (size_t i = 0; i < num_mutants; ++i) { - mutants.push_back(GetMutant(++number_of_mutations_)); + mutants.push_back({/*data=*/GetMutant(++number_of_mutations_)}); } return mutants; } @@ -703,12 +706,12 @@ class ExtraBinariesMock : public CentipedeCallbacks { } // Sets the mutants to different 1-byte values. - std::vector Mutate(const std::vector &inputs, - size_t num_mutants) override { - std::vector mutants{num_mutants}; + std::vector Mutate(const std::vector& inputs, + size_t num_mutants) override { + std::vector mutants(num_mutants); for (auto &mutant : mutants) { - mutant.resize(1); - mutant[0] = ++number_of_mutations_; + mutant.data.resize(1); + mutant.data[0] = ++number_of_mutations_; } return mutants; } @@ -836,13 +839,13 @@ class UndetectedCrashingInputMock : public CentipedeCallbacks { } // Sets the mutants to different 1-byte values. - std::vector Mutate(const std::vector &inputs, - size_t num_mutants) override { - std::vector mutants; + std::vector Mutate(const std::vector& inputs, + size_t num_mutants) override { + std::vector mutants; mutants.reserve(num_mutants); for (size_t i = 0; i < num_mutants; ++i) { // The contents of each mutant is simply its sequential number. - mutants.push_back({static_cast(curr_input_idx_++)}); + mutants.push_back({/*data=*/{static_cast(curr_input_idx_++)}}); } return mutants; } @@ -996,9 +999,9 @@ class FakeCentipedeCallbacksForThreadChecking : public CentipedeCallbacks { return true; } - std::vector Mutate(const std::vector &inputs, - size_t num_mutants) override { - return {num_mutants, {0}}; + std::vector Mutate(const std::vector& inputs, + size_t num_mutants) override { + return {num_mutants, {/*data=*/{0}}}; } bool thread_check_passed() { return thread_check_passed_; } @@ -1051,9 +1054,9 @@ class SetupFailureCallbacks : public CentipedeCallbacks { return false; } - std::vector Mutate(const std::vector &inputs, - size_t num_mutants) override { - return {num_mutants, {0}}; + std::vector Mutate(const std::vector& inputs, + size_t num_mutants) override { + return {num_mutants, {/*data=*/{0}}}; } int execute_count() const { return execute_count_; } @@ -1089,9 +1092,9 @@ class SkippedTestCallbacks : public CentipedeCallbacks { return false; } - std::vector Mutate(const std::vector &inputs, - size_t num_mutants) override { - return {num_mutants, {0}}; + std::vector Mutate(const std::vector& inputs, + size_t num_mutants) override { + return {num_mutants, {/*data=*/{0}}}; } int execute_count() const { return execute_count_; } @@ -1127,9 +1130,9 @@ class IgnoredFailureCallbacks : public CentipedeCallbacks { return false; } - std::vector Mutate(const std::vector &inputs, - size_t num_mutants) override { - return {num_mutants, {0}}; + std::vector Mutate(const std::vector& inputs, + size_t num_mutants) override { + return {num_mutants, {/*data=*/{0}}}; } int execute_count() const { return execute_count_; } @@ -1160,11 +1163,11 @@ TEST_F(CentipedeWithTemporaryLocalDir, UsesProvidedCustomMutator) { CentipedeDefaultCallbacks callbacks(env); const std::vector inputs = {{1}, {2}, {3}, {4}, {5}, {6}}; - const std::vector mutants = callbacks.Mutate( + const std::vector mutants = callbacks.Mutate( GetMutationInputRefsFromDataInputs(inputs), inputs.size()); // The custom mutator just returns the original inputs as mutants. - EXPECT_EQ(inputs, mutants); + EXPECT_EQ(inputs, GetDataFromMutants(mutants)); } TEST_F(CentipedeWithTemporaryLocalDir, FailsOnMisbehavingCustomMutator) { @@ -1193,12 +1196,12 @@ TEST_F(CentipedeWithTemporaryLocalDir, CentipedeDefaultCallbacks callbacks(env); const std::vector inputs = {{1}, {2}, {3}, {4}, {5}, {6}}; - const std::vector mutants = callbacks.Mutate( + const std::vector mutants = callbacks.Mutate( GetMutationInputRefsFromDataInputs(inputs), inputs.size()); // The built-in mutator performs non-trivial mutations. EXPECT_EQ(inputs.size(), mutants.size()); - EXPECT_NE(inputs, mutants); + EXPECT_NE(inputs, GetDataFromMutants(mutants)); } TEST_F(CentipedeWithTemporaryLocalDir, diff --git a/centipede/dispatcher.cc b/centipede/dispatcher.cc index d82cd2834..b11a266a3 100644 --- a/centipede/dispatcher.cc +++ b/centipede/dispatcher.cc @@ -536,7 +536,7 @@ void FuzzTestDispatcherEmitMutant(const void* data, size_t size) { auto* output = GetOutputsBlobSequence(); DispatcherCheck(output != nullptr, "outputs blob sequence must exist"); DispatcherCheck(MutationResult::WriteMutant( - {static_cast(data), size}, *output), + {{static_cast(data), size}}, *output), "failed to write mutant"); } diff --git a/centipede/fuzztest_mutator.cc b/centipede/fuzztest_mutator.cc index 0690f2801..ccacd3e2f 100644 --- a/centipede/fuzztest_mutator.cc +++ b/centipede/fuzztest_mutator.cc @@ -29,7 +29,7 @@ #include "./centipede/byte_array_mutator.h" #include "./centipede/execution_metadata.h" #include "./centipede/knobs.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./common/defs.h" #include "./common/logging.h" #include "./fuzztest/domain_core.h" @@ -42,6 +42,16 @@ namespace { using MutatorDomainBase = decltype(fuzztest::VectorOf(fuzztest::Arbitrary())); +template +bool SampleInsert(const T& cmp_table, size_t& counter) { + static thread_local absl::BitGen bitgen; + counter++; + if (counter <= cmp_table.kTableSize) { + return true; + } + return absl::Uniform(bitgen, 0, counter) < cmp_table.kTableSize; +} + template void InsertCmpEntryIntoIntegerDictionary(const uint8_t* a, const uint8_t* b, TablesOfRecentCompares& cmp_tables) { @@ -57,27 +67,38 @@ void InsertCmpEntryIntoIntegerDictionary(const uint8_t* a, const uint8_t* b, void PopulateCmpEntries(const ExecutionMetadata& metadata, TablesOfRecentCompares& cmp_tables) { // Size limits on the cmp entries to be populated. - static constexpr uint8_t kMaxCmpEntrySize = 15; + static constexpr uint8_t kMaxCmpEntrySize = 128; static constexpr uint8_t kMinCmpEntrySize = 2; + size_t uint16_sample_counter = 0; + size_t uint32_sample_counter = 0; + size_t uint64_sample_counter = 0; + size_t mem_sample_counter = 0; - metadata.ForEachCmpEntry([&cmp_tables](fuzztest::internal::ByteSpan a, - fuzztest::internal::ByteSpan b) { + metadata.ForEachCmpEntry([&](fuzztest::internal::ByteSpan a, + fuzztest::internal::ByteSpan b) { FUZZTEST_CHECK(a.size() == b.size()) << "cmp operands must have the same size"; const size_t size = a.size(); if (size < kMinCmpEntrySize) return; if (size > kMaxCmpEntrySize) return; - if (size == 2) { + if (size == 2 && SampleInsert(cmp_tables.GetMutable(), + uint16_sample_counter)) { InsertCmpEntryIntoIntegerDictionary(a.data(), b.data(), cmp_tables); - } else if (size == 4) { + } else if (size == 4 && + SampleInsert(cmp_tables.GetMutable(), + uint32_sample_counter)) { InsertCmpEntryIntoIntegerDictionary(a.data(), b.data(), cmp_tables); - } else if (size == 8) { + } else if (size == 8 && + SampleInsert(cmp_tables.GetMutable(), + uint64_sample_counter)) { InsertCmpEntryIntoIntegerDictionary(a.data(), b.data(), cmp_tables); } - cmp_tables.GetMutable<0>().Insert(a.data(), b.data(), size); + if (SampleInsert(cmp_tables.GetMutable<0>(), mem_sample_counter)) { + cmp_tables.GetMutable<0>().Insert(a.data(), b.data(), size); + } }); } @@ -139,12 +160,12 @@ void FuzzTestMutator::CrossOver(ByteArray &data, const ByteArray &other) { } } -std::vector FuzzTestMutator::MutateMany( - const std::vector &inputs, size_t num_mutants) { +std::vector FuzzTestMutator::MutateMany( + const std::vector& inputs, size_t num_mutants) { if (inputs.empty()) abort(); auto& cmp_tables = mutation_metadata_->cmp_tables; cmp_tables.resize(inputs.size()); - std::vector mutants; + std::vector mutants; mutants.reserve(num_mutants); for (int i = 0; i < num_mutants; ++i) { auto index = absl::Uniform(prng_, 0, inputs.size()); @@ -152,16 +173,16 @@ std::vector FuzzTestMutator::MutateMany( cmp_tables[index].emplace(/*compact=*/true); PopulateCmpEntries(*inputs[index].metadata, *cmp_tables[index]); } - auto mutant = inputs[index].data; - if (mutant.size() > max_len_) mutant.resize(max_len_); + Mutant mutant = {/*data=*/inputs[index].data}; + if (mutant.data.size() > max_len_) mutant.data.resize(max_len_); if (knobs_.GenerateBool(knob_mutate_or_crossover, prng_())) { // Perform crossover with some other input. It may be the same input. const auto &other_input = inputs[absl::Uniform(prng_, 0, inputs.size())].data; - CrossOver(mutant, other_input); + CrossOver(mutant.data, other_input); } else { domain_->Mutate( - mutant, prng_, + mutant.data, prng_, {/*cmp_tables=*/cmp_tables[index].has_value() ? &*cmp_tables[index] : nullptr}, /*only_shrink=*/false); diff --git a/centipede/fuzztest_mutator.h b/centipede/fuzztest_mutator.h index 8b6846096..ff204f2eb 100644 --- a/centipede/fuzztest_mutator.h +++ b/centipede/fuzztest_mutator.h @@ -22,7 +22,7 @@ #include "./centipede/execution_metadata.h" #include "./centipede/knobs.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./common/defs.h" #include "./fuzztest/internal/table_of_recent_compares.h" @@ -44,8 +44,8 @@ class FuzzTestMutator { ~FuzzTestMutator(); // Takes non-empty `inputs` and produces `num_mutants` mutants. - std::vector MutateMany(const std::vector &inputs, - size_t num_mutants); + std::vector MutateMany(const std::vector& inputs, + size_t num_mutants); // Adds `dict_entries` to the internal mutation dictionary. void AddToDictionary(const std::vector& dict_entries); diff --git a/centipede/fuzztest_mutator_test.cc b/centipede/fuzztest_mutator_test.cc index 327bc836b..a732bcad1 100644 --- a/centipede/fuzztest_mutator_test.cc +++ b/centipede/fuzztest_mutator_test.cc @@ -24,7 +24,7 @@ #include "absl/strings/str_join.h" #include "./centipede/execution_metadata.h" #include "./centipede/knobs.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./common/defs.h" namespace fuzztest::internal { @@ -33,6 +33,7 @@ namespace { using ::testing::AllOf; using ::testing::Each; +using ::testing::Field; using ::testing::IsSupersetOf; using ::testing::Le; using ::testing::SizeIs; @@ -49,10 +50,10 @@ TEST(FuzzTestMutator, DifferentRngSeedsLeadToDifferentMutantSequences) { std::vector mutation_inputs = {{data}}; constexpr size_t kMutantSequenceLength = 100; for (size_t iter = 0; iter < kMutantSequenceLength; iter++) { - const std::vector mutants = + const std::vector mutants = mutator[i].MutateMany(mutation_inputs, 1); ASSERT_EQ(mutants.size(), 1); - res[i].push_back(mutants[0]); + res[i].push_back(mutants[0].data); } } EXPECT_NE(res[0], res[1]); @@ -64,7 +65,7 @@ TEST(FuzzTestMutator, MutateManyWorksWithInputsLargerThanMaxLen) { FuzzTestMutator mutator(knobs, /*seed=*/1); EXPECT_TRUE(mutator.set_max_len(kMaxLen)); constexpr size_t kNumMutantsToGenerate = 10000; - const std::vector mutants = mutator.MutateMany( + const std::vector mutants = mutator.MutateMany( { {/*data=*/{0, 1, 2, 3, 4, 5, 6, 7}}, {/*data=*/{0}}, @@ -74,70 +75,70 @@ TEST(FuzzTestMutator, MutateManyWorksWithInputsLargerThanMaxLen) { }, kNumMutantsToGenerate); - EXPECT_THAT(mutants, - AllOf(SizeIs(kNumMutantsToGenerate), Each(SizeIs(Le(kMaxLen))))); + EXPECT_THAT(mutants, AllOf(SizeIs(kNumMutantsToGenerate), + Each(Field(&Mutant::data, SizeIs(Le(kMaxLen)))))); } TEST(FuzzTestMutator, CrossOverInsertsDataFromOtherInputs) { const Knobs knobs; FuzzTestMutator mutator(knobs, /*seed=*/1); constexpr size_t kNumMutantsToGenerate = 100000; - const std::vector mutants = mutator.MutateMany( + const std::vector mutants = mutator.MutateMany( { {/*data=*/{0, 1, 2, 3}}, {/*data=*/{4, 5, 6, 7}}, }, kNumMutantsToGenerate); - EXPECT_THAT(mutants, IsSupersetOf(std::vector{ - // The entire other input - {4, 5, 6, 7, 0, 1, 2, 3}, - {0, 1, 4, 5, 6, 7, 2, 3}, - {0, 1, 2, 3, 4, 5, 6, 7}, - // The prefix of other input - {4, 5, 6, 0, 1, 2, 3}, - {0, 1, 4, 5, 6, 2, 3}, - {0, 1, 2, 3, 4, 5, 6}, - // The suffix of other input - {5, 6, 7, 0, 1, 2, 3}, - {0, 1, 5, 6, 7, 2, 3}, - {0, 1, 2, 3, 5, 6, 7}, - // The middle of other input - {5, 6, 0, 1, 2, 3}, - {0, 1, 5, 6, 2, 3}, - {0, 1, 2, 3, 5, 6}, - })); + EXPECT_THAT(GetDataFromMutants(mutants), IsSupersetOf(std::vector{ + // The entire other input + {4, 5, 6, 7, 0, 1, 2, 3}, + {0, 1, 4, 5, 6, 7, 2, 3}, + {0, 1, 2, 3, 4, 5, 6, 7}, + // The prefix of other input + {4, 5, 6, 0, 1, 2, 3}, + {0, 1, 4, 5, 6, 2, 3}, + {0, 1, 2, 3, 4, 5, 6}, + // The suffix of other input + {5, 6, 7, 0, 1, 2, 3}, + {0, 1, 5, 6, 7, 2, 3}, + {0, 1, 2, 3, 5, 6, 7}, + // The middle of other input + {5, 6, 0, 1, 2, 3}, + {0, 1, 5, 6, 2, 3}, + {0, 1, 2, 3, 5, 6}, + })); } TEST(FuzzTestMutator, CrossOverOverwritesDataFromOtherInputs) { const Knobs knobs; FuzzTestMutator mutator(knobs, /*seed=*/1); constexpr size_t kNumMutantsToGenerate = 100000; - const std::vector mutants = mutator.MutateMany( + const std::vector mutants = mutator.MutateMany( { {/*data=*/{0, 1, 2, 3, 4, 5, 6, 7}}, {/*data=*/{100, 101, 102, 103}}, }, kNumMutantsToGenerate); - EXPECT_THAT(mutants, IsSupersetOf(std::vector{ - // The entire other input - {100, 101, 102, 103, 4, 5, 6, 7}, - {0, 1, 100, 101, 102, 103, 6, 7}, - {0, 1, 2, 3, 100, 101, 102, 103}, - // The prefix of other input - {100, 101, 102, 3, 4, 5, 6, 7}, - {0, 1, 2, 100, 101, 102, 6, 7}, - {0, 1, 2, 3, 4, 100, 101, 102}, - // The suffix of other input - {101, 102, 103, 3, 4, 5, 6, 7}, - {0, 1, 2, 101, 102, 103, 6, 7}, - {0, 1, 2, 3, 4, 101, 102, 103}, - // The middle of other input - {101, 102, 2, 3, 4, 5, 6, 7}, - {0, 1, 2, 101, 102, 5, 6, 7}, - {0, 1, 2, 3, 4, 5, 101, 102}, - })); + EXPECT_THAT(GetDataFromMutants(mutants), IsSupersetOf(std::vector{ + // The entire other input + {100, 101, 102, 103, 4, 5, 6, 7}, + {0, 1, 100, 101, 102, 103, 6, 7}, + {0, 1, 2, 3, 100, 101, 102, 103}, + // The prefix of other input + {100, 101, 102, 3, 4, 5, 6, 7}, + {0, 1, 2, 100, 101, 102, 6, 7}, + {0, 1, 2, 3, 4, 100, 101, 102}, + // The suffix of other input + {101, 102, 103, 3, 4, 5, 6, 7}, + {0, 1, 2, 101, 102, 103, 6, 7}, + {0, 1, 2, 3, 4, 101, 102, 103}, + // The middle of other input + {101, 102, 2, 3, 4, 5, 6, 7}, + {0, 1, 2, 101, 102, 5, 6, 7}, + {0, 1, 2, 3, 4, 5, 101, 102}, + })); } // Test parameter containing the mutation settings and the expectations of a @@ -181,12 +182,12 @@ TEST_P(MutationStepTest, GeneratesExpectedMutantsAndAvoidsUnexpectedMutants) { const std::vector inputs = { {/*data=*/GetParam().seed_input, /*metadata=*/&metadata}}; for (size_t i = 0; i < GetParam().max_num_iterations; i++) { - const std::vector mutants = mutator.MutateMany(inputs, 1); + const std::vector mutants = mutator.MutateMany(inputs, 1); ASSERT_EQ(mutants.size(), 1); const auto& mutant = mutants[0]; - EXPECT_FALSE(unexpected_mutants.contains(mutant)) - << "Unexpected mutant: {" << absl::StrJoin(mutant, ",") << "}"; - unmatched_expected_mutants.erase(mutant); + EXPECT_FALSE(unexpected_mutants.contains(mutant.data)) + << "Unexpected mutant: {" << absl::StrJoin(mutant.data, ",") << "}"; + unmatched_expected_mutants.erase(mutant.data); if (unmatched_expected_mutants.empty() && i >= GetParam().min_num_iterations) break; @@ -284,22 +285,34 @@ INSTANTIATE_TEST_SUITE_P(InsertFromCmpDictionary, MutationStepTest, Values([] { INSTANTIATE_TEST_SUITE_P(SkipsLongCmpEntry, MutationStepTest, Values([] { MutationStepTestParameter params; params.seed_input = {0}; + ByteArray short_entry; + for (size_t i = 0; i < 5; ++i) { + short_entry.push_back(i); + } params.expected_mutants = { - {0, 1, 2, 3, 4}, + short_entry, }; + ByteArray long_entry; + for (size_t i = 0; i < 129; ++i) { + long_entry.push_back(i); + } params.unexpected_mutants = { - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, + long_entry, }; - params.cmp_data = { - 20, // size - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, // lhs - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, // rhs - 4, // size - 1, 2, 3, 4, // lhs - 1, 2, 3, 4}; // rhs + params.cmp_data.push_back(short_entry.size()); + params.cmp_data.insert(params.cmp_data.end(), + short_entry.begin(), + short_entry.end()); // lhs + params.cmp_data.insert(params.cmp_data.end(), + short_entry.begin(), + short_entry.end()); // rhs + params.cmp_data.push_back(long_entry.size()); + params.cmp_data.insert(params.cmp_data.end(), + long_entry.begin(), + long_entry.end()); // lhs + params.cmp_data.insert(params.cmp_data.end(), + long_entry.begin(), + long_entry.end()); // rhs return params; }())); diff --git a/centipede/minimize_crash.cc b/centipede/minimize_crash.cc index d9fa681e8..2f410469b 100644 --- a/centipede/minimize_crash.cc +++ b/centipede/minimize_crash.cc @@ -26,7 +26,7 @@ #include "absl/synchronization/mutex.h" #include "./centipede/centipede_callbacks.h" #include "./centipede/environment.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/runner_result.h" #include "./centipede/stop.h" #include "./centipede/thread_pool.h" @@ -114,11 +114,11 @@ static void MinimizeCrash(const Environment &env, // discarding all inputs that are too large. // TODO(kcc): modify the Mutate() interface such that max_len can be passed. // - const std::vector mutants = callbacks->Mutate( + const std::vector mutants = callbacks->Mutate( GetMutationInputRefsFromDataInputs(recent_crashers), env.batch_size); std::vector smaller_mutants; for (const auto &m : mutants) { - if (m.size() < min_known_size) smaller_mutants.push_back(m); + if (m.data.size() < min_known_size) smaller_mutants.push_back(m.data); } // Execute all mutants. If a new crasher is found, add it to `queue`. diff --git a/centipede/mutation_input.h b/centipede/mutation_data.h similarity index 64% rename from centipede/mutation_input.h rename to centipede/mutation_data.h index 504c75327..fe0ac5fc6 100644 --- a/centipede/mutation_input.h +++ b/centipede/mutation_data.h @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Data types used for mutation inputs. +// Data types used for mutation. // // This library is for both engine and runner. -#ifndef THIRD_PARTY_CENTIPEDE_MUTATION_INPUT_H_ -#define THIRD_PARTY_CENTIPEDE_MUTATION_INPUT_H_ +#ifndef THIRD_PARTY_CENTIPEDE_MUTATION_DATA_H_ +#define THIRD_PARTY_CENTIPEDE_MUTATION_DATA_H_ #include @@ -48,6 +48,32 @@ inline std::vector GetMutationInputRefsFromDataInputs( return results; } +// Represents a mutation result. +struct Mutant { + // The mutant `data`. + ByteArray data; + + // For testing. + bool operator==(const Mutant& other) const { return data == other.data; } +}; + +// A reference counterpart of `Mutant`. Needed because it can be constructed +// from std::string and/or by the C-only dispatcher without copying the +// underlying data. +struct MutantRef { + ByteSpan data; +}; + +inline std::vector GetDataFromMutants( + const std::vector& mutants) { + std::vector results; + results.reserve(mutants.size()); + for (const auto& mutant : mutants) { + results.push_back(mutant.data); + } + return results; +} + } // namespace fuzztest::internal -#endif // THIRD_PARTY_CENTIPEDE_MUTATION_INPUT_H_ +#endif // THIRD_PARTY_CENTIPEDE_MUTATION_DATA_H_ diff --git a/centipede/mutation_input_test.cc b/centipede/mutation_data_test.cc similarity index 96% rename from centipede/mutation_input_test.cc rename to centipede/mutation_data_test.cc index f1bc7414c..4ccf7ca5b 100644 --- a/centipede/mutation_input_test.cc +++ b/centipede/mutation_data_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include diff --git a/centipede/runner.cc b/centipede/runner.cc index fbf5a2bc0..4460ed3a4 100644 --- a/centipede/runner.cc +++ b/centipede/runner.cc @@ -53,7 +53,7 @@ #include "./centipede/dispatcher_flag_helper.h" #include "./centipede/execution_metadata.h" #include "./centipede/feature.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/runner_interface.h" #include "./centipede/runner_request.h" #include "./centipede/runner_result.h" @@ -327,8 +327,8 @@ void RunnerCallbacks::GetSeeds(std::function seed_callback) { std::string RunnerCallbacks::GetSerializedTargetConfig() { return ""; } bool RunnerCallbacks::Mutate( - const std::vector & /*inputs*/, size_t /*num_mutants*/, - std::function /*new_mutant_callback*/) { + const std::vector& /*inputs*/, size_t /*num_mutants*/, + std::function /*new_mutant_callback*/) { RunnerCheck(!HasCustomMutator(), "Class deriving from RunnerCallbacks must implement Mutate() if " "HasCustomMutator() returns true."); @@ -358,8 +358,8 @@ class LegacyRunnerCallbacks : public RunnerCallbacks { return custom_mutator_cb_ != nullptr; } - bool Mutate(const std::vector &inputs, size_t num_mutants, - std::function new_mutant_callback) override; + bool Mutate(const std::vector& inputs, size_t num_mutants, + std::function new_mutant_callback) override; private: FuzzerTestOneInputCallback test_one_input_cb_; @@ -623,7 +623,7 @@ static int MutateInputsFromShmem(BlobSequence &inputs_blobseq, } if (!callbacks.HasCustomMutator()) return EXIT_SUCCESS; - if (!callbacks.Mutate(input_refs, num_mutants, [&](ByteSpan mutant) { + if (!callbacks.Mutate(input_refs, num_mutants, [&](MutantRef mutant) { MutationResult::WriteMutant(mutant, outputs_blobseq); })) { return EXIT_FAILURE; @@ -632,8 +632,8 @@ static int MutateInputsFromShmem(BlobSequence &inputs_blobseq, } bool LegacyRunnerCallbacks::Mutate( - const std::vector &inputs, size_t num_mutants, - std::function new_mutant_callback) { + const std::vector& inputs, size_t num_mutants, + std::function new_mutant_callback) { if (custom_mutator_cb_ == nullptr) return false; unsigned int seed = GetRandomSeed(); const size_t num_inputs = inputs.size(); @@ -644,7 +644,7 @@ bool LegacyRunnerCallbacks::Mutate( attempt < num_mutants * kAverageMutationAttempts && num_outputs < num_mutants; ++attempt) { - const auto &input_data = inputs[rand_r(&seed) % num_inputs].data; + const auto& input_data = inputs[rand_r(&seed) % num_inputs].data; size_t size = std::min(input_data.size(), max_mutant_size); std::copy(input_data.cbegin(), input_data.cbegin() + size, mutant.begin()); @@ -661,7 +661,8 @@ bool LegacyRunnerCallbacks::Mutate( rand_r(&seed)); } if (new_size == 0) continue; - new_mutant_callback({mutant.data(), new_size}); + if (new_size > max_mutant_size) new_size = max_mutant_size; + new_mutant_callback({{mutant.data(), new_size}}); ++num_outputs; } return true; diff --git a/centipede/runner_cmp_trace.h b/centipede/runner_cmp_trace.h index 66878758b..32f081a55 100644 --- a/centipede/runner_cmp_trace.h +++ b/centipede/runner_cmp_trace.h @@ -18,10 +18,14 @@ // Capturing arguments of CMP instructions, memcmp, and similar. // WARNING: this code needs to have minimal dependencies. +#include + #include #include #include +#include "absl/base/optimization.h" + namespace fuzztest::internal { // Captures up to `kNumItems` different CMP argument pairs. @@ -33,28 +37,47 @@ namespace fuzztest::internal { // // Every new captured pair may overwrite a pair stored previously. // -// Outside of tests, objects of this class will be created in TLS, thus no CTOR. +// Outside of tests, objects of this class will be zero-initialized in TLS, +// thus no CTOR. In tests the objects should be default-initialized. template class CmpTrace { public: // kMaxNumBytesPerValue does not depend on kFixedSize. - static constexpr size_t kMaxNumBytesPerValue = 16; + static constexpr size_t kMaxNumBytesPerValue = 128; static constexpr size_t kNumBytesPerValue = kFixedSize ? kFixedSize : kMaxNumBytesPerValue; // No CTOR - objects will be created in TLS. // Clears `this`. - void Clear() { memset(this, 0, sizeof(*this)); } + void Clear() { to_clear = true; } // Captures one CMP argument pair, as two byte arrays, `size` bytes each. void Capture(uint8_t size, const uint8_t *value0, const uint8_t *value1) { + if (ABSL_PREDICT_FALSE(to_clear)) { + for (size_t i = 0; i < kNumItems; ++i) { + if (sizes_[i] == 0) break; + sizes_[i] = 0; + } + capture_count_ = 0; + to_clear = false; + } if (size > kNumBytesPerValue) size = kNumBytesPerValue; - // We choose a pseudo-random slot each time. - // This way after capturing many pairs we end up with up to `kNumItems` - // pairs which are typically, but not always, the most recent. - rand_seed_ = rand_seed_ * 1103515245 + 12345; - const size_t index = rand_seed_ % kNumItems; + // Fill the initial `kNumItems` pairs sequentially, then randomly overwrite + // previous entries with diminishing probability. + size_t index = capture_count_++; + if (index >= kNumItems) { + if (rand_seed_ == 0) { + // Initialize the random seed (likely) once. + struct timeval tv = {}; + constexpr size_t kUsecInSec = 1000000; + gettimeofday(&tv, nullptr); + rand_seed_ = tv.tv_sec * kUsecInSec + tv.tv_usec; + } + rand_seed_ = rand_seed_ * 1103515245 + 12345; + index = rand_seed_ % capture_count_; + if (index >= kNumItems) return; + } Item& item = items_[index]; sizes_[index] = size; __builtin_memcpy(item.value0, value0, size); @@ -74,12 +97,14 @@ class CmpTrace { // Iterates non-zero CMP pairs. template void ForEachNonZero(Callback callback) { + if (ABSL_PREDICT_FALSE(to_clear)) return; for (size_t i = 0; i < kNumItems; ++i) { const auto size = sizes_[i]; - if (size == 0 || size > kNumBytesPerValue) continue; - sizes_[i] = 0; + if (size == 0) break; + if (size > kNumBytesPerValue) continue; callback(size, items_[i].value0, items_[i].value1); } + to_clear = true; } private: @@ -89,17 +114,22 @@ class CmpTrace { uint8_t value1[kNumBytesPerValue]; }; - // Value sizes of argument pairs. zero-size indicates that the corresponding - // entry is empty. + volatile bool to_clear; + + // Value sizes of argument pairs. Zero-size indicates end of valid entries. // // Marked volatile because of the potential racing between the owning thread - // and the main thread, which is tolerated gracefully. + // and the main thread, which is tolerated gracefully. It is written by only + // the owning thread. volatile uint8_t sizes_[kNumItems]; + + size_t capture_count_; // Values of argument pairs. Item items_[kNumItems]; - // Pseudo-random seed. - size_t rand_seed_; + // Pseudo-random seed from glibc + // (https://en.wikipedia.org/wiki/Linear_congruential_generator). + uint32_t rand_seed_; }; } // namespace fuzztest::internal diff --git a/centipede/runner_cmp_trace_test.cc b/centipede/runner_cmp_trace_test.cc index 492c18f4d..ade09d485 100644 --- a/centipede/runner_cmp_trace_test.cc +++ b/centipede/runner_cmp_trace_test.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include "gmock/gmock.h" @@ -56,10 +57,10 @@ TEST(CmpTrace, T1) { observed_pairs.push_back(cmp_pair); }; - CmpTrace<2, 10> trace2; - CmpTrace<4, 11> trace4; - CmpTrace<8, 12> trace8; - CmpTrace<0, 13> traceN; + CmpTrace<2, 10> trace2 = {}; + CmpTrace<4, 11> trace4 = {}; + CmpTrace<8, 12> trace8 = {}; + CmpTrace<0, 13> traceN = {}; trace2.Clear(); trace4.Clear(); trace8.Clear(); @@ -102,12 +103,12 @@ TEST(CmpTrace, T1) { constexpr uint8_t value0[10] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}; constexpr uint8_t value1[10] = {0, 9, 8, 7, 6, 5, 4, 3, 2, 1}; - constexpr uint8_t long_array[20] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}; + uint8_t long_array[129]; + std::iota(long_array, long_array + 129, 0); traceN.Capture(7, value0, value1); traceN.Capture(3, value0, value1); traceN.Capture(10, value0, value1); - traceN.Capture(20, long_array, long_array); // will be trimmed to 16. + traceN.Capture(129, long_array, long_array); // will be trimmed to 128. observed_pairs.clear(); traceN.ForEachNonZero(callback); EXPECT_THAT(observed_pairs, @@ -115,7 +116,7 @@ TEST(CmpTrace, T1) { TwoArraysToByteVector(value0, value1, 10), TwoArraysToByteVector(value0, value1, 7), TwoArraysToByteVector(value0, value1, 3), - TwoArraysToByteVector(long_array, long_array, 16))); + TwoArraysToByteVector(long_array, long_array, 128))); } } // namespace diff --git a/centipede/runner_interface.h b/centipede/runner_interface.h index 09e2bbd0a..cfbf8499b 100644 --- a/centipede/runner_interface.h +++ b/centipede/runner_interface.h @@ -25,7 +25,7 @@ #include #include "absl/base/nullability.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./common/defs.h" // Typedefs for the libFuzzer API, https://llvm.org/docs/LibFuzzer.html @@ -153,9 +153,9 @@ class RunnerCallbacks { // // TODO(xinhaoyuan): Consider supporting only_shrink to speed up // input shrinking. - virtual bool Mutate(const std::vector &inputs, + virtual bool Mutate(const std::vector& inputs, size_t num_mutants, - std::function new_mutant_callback); + std::function new_mutant_callback); virtual ~RunnerCallbacks() = default; }; diff --git a/centipede/runner_request.cc b/centipede/runner_request.cc index 7d286d30d..6a33cb79e 100644 --- a/centipede/runner_request.cc +++ b/centipede/runner_request.cc @@ -18,7 +18,7 @@ #include #include "./centipede/execution_metadata.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/shared_memory_blob_sequence.h" #include "./common/defs.h" diff --git a/centipede/runner_request.h b/centipede/runner_request.h index 63b90a257..f95189fd2 100644 --- a/centipede/runner_request.h +++ b/centipede/runner_request.h @@ -21,7 +21,7 @@ #include #include "./centipede/execution_metadata.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/shared_memory_blob_sequence.h" #include "./common/defs.h" diff --git a/centipede/runner_result.cc b/centipede/runner_result.cc index 3b726e758..7760fa57a 100644 --- a/centipede/runner_result.cc +++ b/centipede/runner_result.cc @@ -22,6 +22,7 @@ #include "./centipede/execution_metadata.h" #include "./centipede/feature.h" +#include "./centipede/mutation_data.h" #include "./centipede/shared_memory_blob_sequence.h" #include "./common/defs.h" @@ -185,8 +186,8 @@ bool MutationResult::WriteHasCustomMutator(bool has_custom_mutator, reinterpret_cast(&has_custom_mutator)}); } -bool MutationResult::WriteMutant(ByteSpan mutant, BlobSequence &blobseq) { - return blobseq.Write({kTagMutant, mutant.size(), mutant.data()}); +bool MutationResult::WriteMutant(MutantRef mutant, BlobSequence& blobseq) { + return blobseq.Write({kTagMutant, mutant.data.size(), mutant.data.data()}); } bool MutationResult::Read(size_t num_mutants, BlobSequence &blobseq) { @@ -202,7 +203,7 @@ bool MutationResult::Read(size_t num_mutants, BlobSequence &blobseq) { const Blob blob = blobseq.Read(); if (blob.tag != kTagMutant) return false; if (blob.size == 0) break; - mutants_.emplace_back(blob.data, blob.data + blob.size); + mutants_.push_back({ByteArray{blob.data, blob.data + blob.size}}); } return true; } diff --git a/centipede/runner_result.h b/centipede/runner_result.h index 1b94f3187..8266103a6 100644 --- a/centipede/runner_result.h +++ b/centipede/runner_result.h @@ -25,6 +25,7 @@ #include "./centipede/execution_metadata.h" #include "./centipede/feature.h" +#include "./centipede/mutation_data.h" #include "./centipede/shared_memory_blob_sequence.h" #include "./common/defs.h" @@ -217,7 +218,7 @@ class MutationResult { BlobSequence& blobseq); // Writes one mutant to `blobseq`. Returns true iff successful. - static bool WriteMutant(ByteSpan mutant, BlobSequence& blobseq); + static bool WriteMutant(MutantRef mutant, BlobSequence& blobseq); // Reads whether the target has a custom mutator, and if so, reads at most // `num_mutants` mutants from `blobseq`. Returns true iff successful. @@ -227,13 +228,13 @@ class MutationResult { int exit_code() const { return exit_code_; } int& exit_code() { return exit_code_; } bool has_custom_mutator() const { return has_custom_mutator_; } - const std::vector& mutants() const& { return mutants_; } - std::vector&& mutants() && { return std::move(mutants_); } + const std::vector& mutants() const& { return mutants_; } + std::vector&& mutants() && { return std::move(mutants_); } private: int exit_code_ = EXIT_SUCCESS; bool has_custom_mutator_ = false; - std::vector mutants_; + std::vector mutants_; }; } // namespace fuzztest::internal diff --git a/centipede/runner_result_test.cc b/centipede/runner_result_test.cc index 864401a62..541494729 100644 --- a/centipede/runner_result_test.cc +++ b/centipede/runner_result_test.cc @@ -29,6 +29,7 @@ #include "gtest/gtest.h" #include "./centipede/execution_metadata.h" #include "./centipede/feature.h" +#include "./centipede/mutation_data.h" #include "./centipede/shared_memory_blob_sequence.h" #include "./common/defs.h" #include "./common/test_util.h" @@ -213,9 +214,9 @@ TEST(MutationResult, WriteThenRead) { // Write a mutation result. ASSERT_TRUE(MutationResult::WriteHasCustomMutator(true, blobseq)); - ASSERT_TRUE(MutationResult::WriteMutant({1, 2, 3}, blobseq)); - ASSERT_TRUE(MutationResult::WriteMutant({4, 5, 6}, blobseq)); - ASSERT_TRUE(MutationResult::WriteMutant({7, 8, 9}, blobseq)); + ASSERT_TRUE(MutationResult::WriteMutant({{1, 2, 3}}, blobseq)); + ASSERT_TRUE(MutationResult::WriteMutant({{4, 5, 6}}, blobseq)); + ASSERT_TRUE(MutationResult::WriteMutant({{7, 8, 9}}, blobseq)); blobseq.Reset(); MutationResult mutation_result; @@ -224,7 +225,7 @@ TEST(MutationResult, WriteThenRead) { EXPECT_TRUE(mutation_result.has_custom_mutator()); EXPECT_THAT( mutation_result.mutants(), - ElementsAre(ByteArray{1, 2, 3}, ByteArray{4, 5, 6}, ByteArray{7, 8, 9})); + ElementsAre(Mutant{{1, 2, 3}}, Mutant{{4, 5, 6}}, Mutant{{7, 8, 9}})); } TEST(ExecutionResult, ReadResultSucceedsOnlyWithInputBegin) { diff --git a/centipede/test_coverage_util.h b/centipede/test_coverage_util.h index c06a9bf7a..4a61ab35e 100644 --- a/centipede/test_coverage_util.h +++ b/centipede/test_coverage_util.h @@ -25,7 +25,7 @@ #include "./centipede/corpus.h" #include "./centipede/environment.h" #include "./centipede/feature.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/runner_result.h" #include "./common/defs.h" namespace fuzztest::internal { @@ -50,8 +50,8 @@ class TestCallbacks : public CentipedeCallbacks { FUZZTEST_CHECK_EQ(EXIT_SUCCESS, result); return true; } - std::vector Mutate(const std::vector &inputs, - size_t num_mutants) override { + std::vector Mutate(const std::vector& inputs, + size_t num_mutants) override { return {}; } }; diff --git a/centipede/testing/BUILD b/centipede/testing/BUILD index ffffb600e..438e7e89d 100644 --- a/centipede/testing/BUILD +++ b/centipede/testing/BUILD @@ -83,7 +83,7 @@ cc_binary( "@abseil-cpp//absl/flags:flag", "@abseil-cpp//absl/flags:parse", "@com_google_fuzztest//centipede:centipede_runner_no_main", - "@com_google_fuzztest//centipede:mutation_input", + "@com_google_fuzztest//centipede:mutation_data", "@com_google_fuzztest//common:defs", ], ) diff --git a/centipede/testing/async_failing_target.cc b/centipede/testing/async_failing_target.cc index e66ac8bf4..3d81ebcd3 100644 --- a/centipede/testing/async_failing_target.cc +++ b/centipede/testing/async_failing_target.cc @@ -33,7 +33,7 @@ class AsyncFailingTargetRunnerCallbacks bool Mutate(const std::vector& inputs, size_t num_mutants, - std::function + std::function new_mutant_callback) override { if (to_fail_in_mutation) { fprintf(stderr, "Fail in mutation\n"); diff --git a/centipede/testing/fuzz_target_with_custom_mutator.cc b/centipede/testing/fuzz_target_with_custom_mutator.cc index a95dbf84d..cef55e269 100644 --- a/centipede/testing/fuzz_target_with_custom_mutator.cc +++ b/centipede/testing/fuzz_target_with_custom_mutator.cc @@ -19,7 +19,7 @@ #include "absl/base/nullability.h" #include "absl/flags/flag.h" #include "absl/flags/parse.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/runner_interface.h" #include "./common/defs.h" @@ -38,12 +38,11 @@ class CustomMutatorRunnerCallbacks bool Mutate(const std::vector& inputs, size_t num_mutants, - std::function new_mutant_callback) override { - size_t i = 0; - for (fuzztest::internal::MutationInputRef input : inputs) { - if (i++ >= num_mutants) break; + std::function + new_mutant_callback) override { + for (size_t i = 0; i < inputs.size() && i < num_mutants; ++i) { // Just return the original input as a mutant. - new_mutant_callback(input.data); + new_mutant_callback({inputs[i].data}); } return true; } diff --git a/fuzztest/internal/BUILD b/fuzztest/internal/BUILD index abb0a3fa0..da4e80609 100644 --- a/fuzztest/internal/BUILD +++ b/fuzztest/internal/BUILD @@ -78,7 +78,7 @@ cc_library( "@com_google_fuzztest//centipede:environment", "@com_google_fuzztest//centipede:execution_metadata", "@com_google_fuzztest//centipede:fuzztest_mutator", - "@com_google_fuzztest//centipede:mutation_input", + "@com_google_fuzztest//centipede:mutation_data", "@com_google_fuzztest//centipede:runner_result", "@com_google_fuzztest//centipede:stop", "@com_google_fuzztest//centipede:workdir", diff --git a/fuzztest/internal/centipede_adaptor.cc b/fuzztest/internal/centipede_adaptor.cc index 9bb0256b2..c06cd521c 100644 --- a/fuzztest/internal/centipede_adaptor.cc +++ b/fuzztest/internal/centipede_adaptor.cc @@ -70,7 +70,7 @@ #include "./centipede/environment.h" #include "./centipede/execution_metadata.h" #include "./centipede/fuzztest_mutator.h" -#include "./centipede/mutation_input.h" +#include "./centipede/mutation_data.h" #include "./centipede/runner_interface.h" #include "./centipede/runner_result.h" #include "./centipede/stop.h" @@ -516,10 +516,8 @@ class CentipedeAdaptorRunnerCallbacks bool HasCustomMutator() const override { return true; } - bool Mutate(const std::vector& inputs, - size_t num_mutants, - std::function - new_mutant_callback) override { + bool Mutate(const std::vector& inputs, size_t num_mutants, + std::function new_mutant_callback) override { if (inputs.empty()) return false; cmp_tables.resize(inputs.size()); absl::Cleanup cmp_tables_cleaner = [this]() { cmp_tables.clear(); }; @@ -556,7 +554,7 @@ class CentipedeAdaptorRunnerCallbacks fuzzer_impl_.params_domain_.SerializeCorpus(mutant.args)); } new_mutant_callback( - {(unsigned char*)mutant_data.data(), mutant_data.size()}); + {{(unsigned char*)mutant_data.data(), mutant_data.size()}}); } return true; } diff --git a/fuzztest/internal/table_of_recent_compares.h b/fuzztest/internal/table_of_recent_compares.h index ed59b71ef..742bf55e5 100644 --- a/fuzztest/internal/table_of_recent_compares.h +++ b/fuzztest/internal/table_of_recent_compares.h @@ -1,4 +1,3 @@ - // Copyright 2022 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License");