From 86502d129f223be4172e5587eeadfaf7352fe558 Mon Sep 17 00:00:00 2001 From: Chris Leary Date: Sat, 4 Jan 2025 18:41:23 -0800 Subject: [PATCH] [DSLX:run_routines] Enable quickcheck to be exhaustive. --- .bazelrc | 1 + docs_src/dslx_reference.md | 20 ++++- xls/data_structures/inline_bitmap.h | 36 +++++++++ xls/dslx/fmt/ast_fmt.cc | 17 ++-- xls/dslx/frontend/ast.cc | 32 ++++++-- xls/dslx/frontend/ast.h | 45 +++++++++-- xls/dslx/frontend/ast_cloner.cc | 2 +- xls/dslx/frontend/parser.cc | 52 ++++++------ xls/dslx/frontend/parser.h | 3 + xls/dslx/run_routines/run_comparator.h | 1 + xls/dslx/run_routines/run_routines.cc | 71 ++++++++++++++-- xls/dslx/run_routines/run_routines.h | 7 +- xls/dslx/run_routines/run_routines_test.cc | 89 ++++++++++++++++++--- xls/dslx/tests/errors/error_modules_test.py | 2 +- xls/ir/bits.h | 4 + xls/ir/value.cc | 22 +++++ xls/ir/value.h | 1 + xls/ir/value_test.cc | 33 +++++++- 18 files changed, 367 insertions(+), 71 deletions(-) diff --git a/.bazelrc b/.bazelrc index 579cb46908..e9e1cefeb2 100644 --- a/.bazelrc +++ b/.bazelrc @@ -25,6 +25,7 @@ build --copt "-Wextra" --host_copt "-Wextra" # Turn some specific warnings into errors. build --copt "-Werror=switch" --host_copt "-Werror=switch" +build --copt "-Werror=return-type" --host_copt "-Werror=return-type" # ... and disable the warnings we're not interested in. build --copt "-Wno-sign-compare" --host_copt "-Wno-sign-compare" diff --git a/docs_src/dslx_reference.md b/docs_src/dslx_reference.md index 77ed5f9cae..0fd9098cdd 100644 --- a/docs_src/dslx_reference.md +++ b/docs_src/dslx_reference.md @@ -2052,7 +2052,7 @@ implementation from above: // Reversing a value twice gets you the original value. #[quickcheck] -fn prop_double_reverse(x: u32) -> bool { +fn prop_double_bitreverse(x: u32) -> bool { x == rev(rev(x)) } ``` @@ -2078,6 +2078,24 @@ for production can be found in the execution log. For determinism, the DSLX interpreter should be run with the `seed` flag: `./interpreter_main --seed=1234 ` +#### Exhaustive QuickCheck + +For small domains the quickcheck directive can also be placed in exhaustive mode via the `exhaustive` directive: + +```dslx +// `u8` space is small enough to check exhaustively. +#[quickcheck(exhaustive)] +fn prop_double_bitreverse(x: u8) -> bool { + x == rev(rev(x)) +} +``` + +This is useful when there are small domains where we would prefer exhaustive +stimulus over randomized stimulus. Note that as the space becomes large, +exhaustive concrete-stimulus-based testing becomes implausible, and users should +consider attempting to prove the QuickCheck formally via the +`prove_quickcheck_main` tool. + [hughes-paper]: https://www.cs.tufts.edu/~nr/cs257/archive/john-hughes/quick.pdf ## Communicating Sequential Processes (AKA procs) diff --git a/xls/data_structures/inline_bitmap.h b/xls/data_structures/inline_bitmap.h index 6860e6e2c0..eb338b6663 100644 --- a/xls/data_structures/inline_bitmap.h +++ b/xls/data_structures/inline_bitmap.h @@ -352,6 +352,42 @@ class InlineBitmap { absl::InlinedVector data_; }; +class BitmapView { + public: + explicit BitmapView(const InlineBitmap& bitmap, + std::optional start_bit = std::nullopt, + std::optional bit_count = std::nullopt) + : bitmap_(bitmap) { + start_bit_ = start_bit.value_or(0); + CHECK_LE(start_bit_, bitmap_.bit_count()); + bit_count_ = bit_count.value_or(bitmap_.bit_count() - start_bit_); + } + + bool Get(int64_t bit_index) const { + CHECK_LT(bit_index, bit_count_); + return bitmap_.Get(start_bit_ + bit_index); + } + + BitmapView Slice(int64_t start_bit, int64_t bit_count) const { + return BitmapView(bitmap_, start_bit_ + start_bit, bit_count); + } + + InlineBitmap ToBitmap() const { + InlineBitmap result(bit_count_, false); + for (int64_t i = 0; i < bit_count_; ++i) { + result.Set(i, Get(i)); + } + return result; + } + + int64_t bit_count() const { return bit_count_; } + + private: + const InlineBitmap& bitmap_; + int64_t start_bit_; + int64_t bit_count_; +}; + } // namespace xls #endif // XLS_DATA_STRUCTURES_INLINE_BITMAP_H_ diff --git a/xls/dslx/fmt/ast_fmt.cc b/xls/dslx/fmt/ast_fmt.cc index 2c462f098b..278c3655f9 100644 --- a/xls/dslx/fmt/ast_fmt.cc +++ b/xls/dslx/fmt/ast_fmt.cc @@ -2373,11 +2373,18 @@ DocRef Formatter::Format(const TestProc& n) { DocRef Formatter::Format(const QuickCheck& n) { std::vector pieces; - if (std::optional test_count = n.test_count()) { - pieces.push_back(arena_.MakeText( - absl::StrFormat("#[quickcheck(test_count=%d)]", test_count.value()))); - } else { - pieces.push_back(arena_.MakeText("#[quickcheck]")); + switch (n.test_cases().tag()) { + case QuickCheckTestCasesTag::kExhaustive: + pieces.push_back(arena_.MakeText("#[quickcheck(exhaustive)]")); + break; + case QuickCheckTestCasesTag::kCounted: + if (n.test_cases().count().has_value()) { + pieces.push_back(arena_.MakeText(absl::StrFormat( + "#[quickcheck(test_count=%d)]", *n.test_cases().count()))); + } else { + pieces.push_back(arena_.MakeText("#[quickcheck]")); + } + break; } pieces.push_back(arena_.hard_line()); pieces.push_back(Format(*n.fn())); diff --git a/xls/dslx/frontend/ast.cc b/xls/dslx/frontend/ast.cc index 01260df26c..dc5f590a46 100644 --- a/xls/dslx/frontend/ast.cc +++ b/xls/dslx/frontend/ast.cc @@ -2167,22 +2167,40 @@ VerbatimNode::~VerbatimNode() = default; // -- class QuickCheck +std::string QuickCheckTestCases::ToString() const { + switch (tag()) { + case QuickCheckTestCasesTag::kExhaustive: + return "exhaustive"; + case QuickCheckTestCasesTag::kCounted: + if (count().has_value()) { + return absl::StrFormat("test_count=%d", count().value()); + } + return absl::StrFormat("test_count=default=%d", kDefaultTestCount); + } +} + QuickCheck::QuickCheck(Module* owner, Span span, Function* fn, - std::optional test_count) + QuickCheckTestCases test_cases) : AstNode(owner), span_(std::move(span)), fn_(fn), - test_count_(test_count) {} + test_cases_(test_cases) {} QuickCheck::~QuickCheck() = default; std::string QuickCheck::ToString() const { - std::string test_count_str; - if (test_count_.has_value()) { - test_count_str = absl::StrFormat("(test_count=%d)", *test_count_); + std::string spec_str; + switch (test_cases_.tag()) { + case QuickCheckTestCasesTag::kExhaustive: + spec_str = "(exhaustive)"; + break; + case QuickCheckTestCasesTag::kCounted: + if (test_cases_.count().has_value()) { + spec_str = absl::StrFormat("(test_count=%d)", *test_cases_.count()); + } + break; } - return absl::StrFormat("#[quickcheck%s]\n%s", test_count_str, - fn_->ToString()); + return absl::StrFormat("#[quickcheck%s]\n%s", spec_str, fn_->ToString()); } // -- class TupleIndex diff --git a/xls/dslx/frontend/ast.h b/xls/dslx/frontend/ast.h index 2bd7e9549e..c57dcae6ba 100644 --- a/xls/dslx/frontend/ast.h +++ b/xls/dslx/frontend/ast.h @@ -3033,15 +3033,47 @@ class TestFunction : public AstNode { Function& fn_; }; +enum class QuickCheckTestCasesTag { + kExhaustive, + kCounted, +}; + +// Describes the test cases that should be run for a quickcheck test function -- +// they can be either counted or exhaustive, and for counted we have a default +// count if the user doesn't explicitly specify a count. +class QuickCheckTestCases { + public: + // The number of test cases we run if a count is not explicitly specified. + static constexpr int64_t kDefaultTestCount = 1000; + + static QuickCheckTestCases Exhaustive() { + return QuickCheckTestCases(QuickCheckTestCasesTag::kExhaustive); + } + static QuickCheckTestCases Counted(std::optional count) { + return QuickCheckTestCases(QuickCheckTestCasesTag::kCounted, count); + } + + std::string ToString() const; + + QuickCheckTestCasesTag tag() const { return tag_; } + std::optional count() const { return count_; } + + private: + explicit QuickCheckTestCases(QuickCheckTestCasesTag tag, + std::optional count = std::nullopt) + : tag_(tag), count_(count) {} + + QuickCheckTestCasesTag tag_; + std::optional count_; +}; + // Represents a function to be quick-check'd. class QuickCheck : public AstNode { public: static std::string_view GetDebugTypeName() { return "quickcheck"; } - static constexpr int64_t kDefaultTestCount = 1000; - QuickCheck(Module* owner, Span span, Function* fn, - std::optional test_count = std::nullopt); + QuickCheckTestCases test_cases); ~QuickCheck() override; @@ -3062,17 +3094,14 @@ class QuickCheck : public AstNode { const std::string& identifier() const { return fn_->identifier(); } Function* fn() const { return fn_; } - int64_t GetTestCountOrDefault() const { - return test_count_ ? *test_count_ : kDefaultTestCount; - } - std::optional test_count() const { return test_count_; } + QuickCheckTestCases test_cases() const { return test_cases_; } std::optional GetSpan() const override { return span_; } const Span& span() const { return span_; } private: Span span_; Function* fn_; - std::optional test_count_; + QuickCheckTestCases test_cases_; }; // Represents an index into a tuple, e.g., "(u32:7, u32:8).1". diff --git a/xls/dslx/frontend/ast_cloner.cc b/xls/dslx/frontend/ast_cloner.cc index 08875082b6..7e22083ed4 100644 --- a/xls/dslx/frontend/ast_cloner.cc +++ b/xls/dslx/frontend/ast_cloner.cc @@ -684,7 +684,7 @@ class AstCloner : public AstNodeVisitor { XLS_RETURN_IF_ERROR(VisitChildren(n)); old_to_new_[n] = module_->Make( n->GetSpan().value(), down_cast(old_to_new_.at(n->fn())), - n->test_count()); + n->test_cases()); return absl::OkStatus(); } diff --git a/xls/dslx/frontend/parser.cc b/xls/dslx/frontend/parser.cc index 841435ad67..69ed07276a 100644 --- a/xls/dslx/frontend/parser.cc +++ b/xls/dslx/frontend/parser.cc @@ -1704,34 +1704,34 @@ absl::StatusOr Parser::ParseFunctionInternal( return f; } +absl::StatusOr Parser::ParseQuickCheckConfig() { + XLS_RETURN_IF_ERROR(DropTokenOrError(TokenKind::kOParen)); + XLS_ASSIGN_OR_RETURN(Token tok, PopTokenOrError(TokenKind::kIdentifier)); + if (tok.GetValue() == "exhaustive") { + XLS_RETURN_IF_ERROR(DropTokenOrError(TokenKind::kCParen)); + return QuickCheckTestCases::Exhaustive(); + } + if (tok.GetValue() == "test_count") { + XLS_RETURN_IF_ERROR(DropTokenOrError(TokenKind::kEquals)); + XLS_ASSIGN_OR_RETURN(Token tok, PopTokenOrError(TokenKind::kNumber)); + XLS_ASSIGN_OR_RETURN(int64_t count, tok.GetValueAsInt64()); + XLS_RETURN_IF_ERROR(DropTokenOrError(TokenKind::kCParen)); + return QuickCheckTestCases::Counted(count); + } + return ParseErrorStatus(tok.span(), + "Expected 'exhaustive' or 'test_count' in " + "quickcheck directive"); +} + absl::StatusOr Parser::ParseQuickCheck( absl::flat_hash_map* name_to_fn, Bindings& bindings, const Pos& hash_pos) { - std::optional test_count; - XLS_ASSIGN_OR_RETURN(bool peek_is_paren, PeekTokenIs(TokenKind::kOParen)); - if (peek_is_paren) { // Config is specified. - DropTokenOrDie(); - Span config_name_span; - XLS_ASSIGN_OR_RETURN(std::string config_name, - PopIdentifierOrError(&config_name_span)); - XLS_RETURN_IF_ERROR(DropTokenOrError(TokenKind::kEquals)); - if (config_name == "test_count") { - XLS_ASSIGN_OR_RETURN(Token count_token, - PopTokenOrError(TokenKind::kNumber)); - XLS_ASSIGN_OR_RETURN(test_count, count_token.GetValueAsInt64()); - if (test_count <= 0) { - return ParseErrorStatus( - count_token.span(), - absl::StrFormat("Number of tests should be > 0, got %d", - *test_count)); - } - XLS_RETURN_IF_ERROR(DropTokenOrError(TokenKind::kCParen)); - } else { - return ParseErrorStatus( - config_name_span, - absl::StrFormat("Unknown configuration key in directive: '%s'", - config_name)); - } + std::optional test_cases; + XLS_ASSIGN_OR_RETURN(bool peek_is_oparen, PeekTokenIs(TokenKind::kOParen)); + if (peek_is_oparen) { // Config is specified. + XLS_ASSIGN_OR_RETURN(test_cases, ParseQuickCheckConfig()); + } else { + test_cases = QuickCheckTestCases::Counted(std::nullopt); } XLS_RETURN_IF_ERROR(DropTokenOrError(TokenKind::kCBrack)); @@ -1739,7 +1739,7 @@ absl::StatusOr Parser::ParseQuickCheck( Function * fn, ParseFunction(GetPos(), /*is_public=*/false, bindings, name_to_fn)); const Span quickcheck_span(hash_pos, fn->span().limit()); - return module_->Make(quickcheck_span, fn, test_count); + return module_->Make(quickcheck_span, fn, test_cases.value()); } absl::StatusOr Parser::ParseTupleRemainder(const Pos& start_pos, diff --git a/xls/dslx/frontend/parser.h b/xls/dslx/frontend/parser.h index a9370f0306..1dec3bc3f6 100644 --- a/xls/dslx/frontend/parser.h +++ b/xls/dslx/frontend/parser.h @@ -581,6 +581,9 @@ class Parser : public TokenParser { absl::flat_hash_map* name_to_fn, Bindings& bindings, const Pos& hash_pos); + // Parses the test count configuration for a quickcheck directive. + absl::StatusOr ParseQuickCheckConfig(); + // Parses a module-level attribute -- cursor should be over the open bracket. // // Side-effect: module_ is tagged with the parsed attribute on success. diff --git a/xls/dslx/run_routines/run_comparator.h b/xls/dslx/run_routines/run_comparator.h index ce542f5ef0..4dda7ac2fd 100644 --- a/xls/dslx/run_routines/run_comparator.h +++ b/xls/dslx/run_routines/run_comparator.h @@ -75,6 +75,7 @@ class RunComparator : public AbstractRunComparator { private: XLS_FRIEND_TEST(RunRoutinesTest, TestInvokedFunctionDoesJit); XLS_FRIEND_TEST(RunRoutinesTest, QuickcheckInvokedFunctionDoesJit); + XLS_FRIEND_TEST(RunRoutinesTest, QuickcheckExhaustive); XLS_FRIEND_TEST(RunRoutinesTest, NoSeedStillQuickChecks); absl::flat_hash_map> jit_cache_; diff --git a/xls/dslx/run_routines/run_routines.cc b/xls/dslx/run_routines/run_routines.cc index 9b4cb052db..63109fca76 100644 --- a/xls/dslx/run_routines/run_routines.cc +++ b/xls/dslx/run_routines/run_routines.cc @@ -270,15 +270,60 @@ static bool TestMatchesFilter(std::string_view test_name, return RE2::FullMatch(test_name, *test_filter); } +// Populates a value from a given tuple type with a flat bit contents given by +// `i` -- this is useful for exhaustively iterating through a space to populate +// an aggregate value. +// +// Precondition: tuple_type->GetFlatBitCount() must be <= 64 so we have enough +// data in `i` to populate it. +static std::vector MakeFromUint64(xls::TupleType* tuple_type, + uint64_t i) { + // We turn the uint64_t contents into a bit vector of the flat bit count of + // the tuple type and populate that tuple type from the bit string. + InlineBitmap bitmap = + InlineBitmap::FromWord(i, tuple_type->GetFlatBitCount()); + BitmapView view(bitmap); + Value value = ZeroOfType(tuple_type); + CHECK_OK(value.PopulateFrom(view)); + return value.GetElements().value(); +}; + absl::StatusOr DoQuickCheck( xls::Function* xls_function, std::string_view ir_name, - AbstractRunComparator* run_comparator, int64_t seed, int64_t num_tests) { + AbstractRunComparator* run_comparator, int64_t seed, + QuickCheckTestCases test_cases) { QuickCheckResults results; std::minstd_rand rng_engine(seed); + xls::TupleType* param_tuple = xls_function->package()->GetTupleType( + xls_function->GetType()->parameters()); + + int64_t num_tests; + std::function(int64_t)> make_arg_set; + switch (test_cases.tag()) { + case QuickCheckTestCasesTag::kExhaustive: { + int64_t parameter_bit_count = param_tuple->GetFlatBitCount(); + if (parameter_bit_count > 48) { + return absl::InvalidArgumentError( + absl::StrFormat("Cannot run an exhaustive quickcheck for `%s` " + "because it has too large a parameter bit count; " + "got: %d", + xls_function->name(), parameter_bit_count)); + } + num_tests = int64_t{1} << parameter_bit_count; + make_arg_set = [&](int64_t i) { return MakeFromUint64(param_tuple, i); }; + break; + } + case QuickCheckTestCasesTag::kCounted: + num_tests = + test_cases.count().value_or(QuickCheckTestCases::kDefaultTestCount); + make_arg_set = [&](int64_t) { + return RandomFunctionArguments(xls_function, rng_engine); + }; + break; + } for (int i = 0; i < num_tests; i++) { - results.arg_sets.push_back( - RandomFunctionArguments(xls_function, rng_engine)); + results.arg_sets.push_back(make_arg_set(i)); // TODO(https://github.com/google/xls/issues/506): 2021-10-15 // Assertion failures should work out, but we should consciously decide // if/how we want to dump traces when running QuickChecks (always, for @@ -351,7 +396,7 @@ static absl::Status RunQuickCheck(AbstractRunComparator* run_comparator, XLS_ASSIGN_OR_RETURN( QuickCheckResults qc_results, DoQuickCheck(qc_fn.ir_function, qc_fn.ir_name, run_comparator, seed, - quickcheck->GetTestCountOrDefault())); + quickcheck->test_cases())); const auto& [arg_sets, results] = qc_results; XLS_ASSIGN_OR_RETURN(Bits last_result, results.back().GetBitsWithStatus()); if (!last_result.IsZero()) { @@ -426,7 +471,7 @@ static absl::Status RunQuickChecksIfJitEnabled( any_quicktest_run = true; } std::cerr << "[ RUN QUICKCHECK ] " << quickcheck_name - << " count: " << quickcheck->GetTestCountOrDefault() << "\n"; + << " cases: " << quickcheck->test_cases().ToString() << "\n"; const absl::Status status = RunQuickCheck(run_comparator, ir_package, quickcheck, type_info, *seed); const absl::Duration duration = absl::Now() - test_case_start; @@ -459,9 +504,15 @@ absl::StatusOr ParseAndProve( const absl::Time start = absl::Now(); TestResultData result(start, /*test_cases=*/{}); + std::unique_ptr vfs; + if (options.vfs_factory != nullptr) { + vfs = options.vfs_factory(); + } else { + vfs = std::make_unique(); + } auto import_data = CreateImportData(options.dslx_stdlib_path, options.dslx_paths, - options.warnings, std::make_unique()); + options.warnings, std::move(vfs)); FileTable& file_table = import_data.file_table(); absl::StatusOr tm = ParseAndTypecheck(program, filename, module_name, &import_data); @@ -638,9 +689,15 @@ absl::StatusOr AbstractTestRunner::ParseAndTest( const absl::Time start = absl::Now(); TestResultData result(start, /*test_cases=*/{}); + std::unique_ptr vfs; + if (options.vfs_factory != nullptr) { + vfs = options.vfs_factory(); + } else { + vfs = std::make_unique(); + } auto import_data = CreateImportData(options.dslx_stdlib_path, options.dslx_paths, - options.warnings, std::make_unique()); + options.warnings, std::move(vfs)); FileTable& file_table = import_data.file_table(); absl::StatusOr tm = diff --git a/xls/dslx/run_routines/run_routines.h b/xls/dslx/run_routines/run_routines.h index 5aa732926a..5dd090e783 100644 --- a/xls/dslx/run_routines/run_routines.h +++ b/xls/dslx/run_routines/run_routines.h @@ -108,6 +108,8 @@ struct ParseAndTestOptions { WarningKindSet warnings = kDefaultWarningsSet; bool trace_channels = false; std::optional max_ticks; + std::function()> vfs_factory = + nullptr; }; // As above, but a subset of the options required for the ParseAndProve() @@ -118,6 +120,8 @@ struct ParseAndProveOptions { const RE2* test_filter = nullptr; bool warnings_as_errors = true; WarningKindSet warnings = kDefaultWarningsSet; + std::function()> vfs_factory = + nullptr; }; enum class TestResult : uint8_t { @@ -290,7 +294,8 @@ struct QuickCheckResults { // length of the returned vectors may be < 1000). absl::StatusOr DoQuickCheck( xls::Function* xls_function, std::string_view ir_name, - AbstractRunComparator* run_comparator, int64_t seed, int64_t num_tests); + AbstractRunComparator* run_comparator, int64_t seed, + QuickCheckTestCases test_cases); } // namespace xls::dslx diff --git a/xls/dslx/run_routines/run_routines_test.cc b/xls/dslx/run_routines/run_routines_test.cc index c4dc9b9d6f..63f9aba118 100644 --- a/xls/dslx/run_routines/run_routines_test.cc +++ b/xls/dslx/run_routines/run_routines_test.cc @@ -167,6 +167,73 @@ fn trivial(x: u5) -> bool { id(true) } EXPECT_EQ(jit_comparator.jit_cache_.begin()->first, "__test__trivial"); } +// A simple exhaustive quickcheck that passes for all values. +TEST_P(RunRoutinesTest, QuickcheckExhaustive) { + constexpr const char* kProgram = R"( +fn id(x: bool) -> bool { x } + +#[quickcheck(exhaustive)] +fn trivial(x: u2) -> bool { id(true) } +)"; + constexpr const char* kModuleName = "test"; + constexpr const char* kFilename = "test.x"; + RunComparator jit_comparator(CompareMode::kJit); + ParseAndTestOptions options; + options.run_comparator = &jit_comparator; + XLS_ASSERT_OK_AND_ASSIGN( + TestResultData result, + ParseAndTest(kProgram, kModuleName, kFilename, options)); + + EXPECT_THAT(result, IsTestResult(TestResult::kAllPassed, 1, 0, 0)); + + ASSERT_EQ(jit_comparator.jit_cache_.size(), 1); + EXPECT_EQ(jit_comparator.jit_cache_.begin()->first, "__test__trivial"); +} + +// An exhaustive quickcheck that fails just for one value in a decently large +// space. +TEST_P(RunRoutinesTest, QuickcheckExhaustiveFail) { + constexpr std::string_view kProgram = R"( +#[quickcheck(exhaustive)] +fn trivial(x: u11) -> bool { x != u11::MAX } +)"; + constexpr const char* kModuleName = "test"; + constexpr const char* kFilename = "test.x"; + RunComparator jit_comparator(CompareMode::kJit); + ParseAndTestOptions options; + options.run_comparator = &jit_comparator; + options.vfs_factory = [kProgram] { + return std::make_unique(kProgram, "test.x"); + }; + XLS_ASSERT_OK_AND_ASSIGN( + TestResultData result, + ParseAndTest(kProgram, kModuleName, kFilename, options)); + EXPECT_THAT(result, IsTestResult(TestResult::kSomeFailed, /*test_cases=*/1, + /*ran_count=*/0, /*failed_count=*/1)); +} + +// An exhaustive quickcheck that fails just for one value in a decently large +// space using a two tuple of params. +TEST_P(RunRoutinesTest, QuickcheckExhaustive2ParamFail) { + constexpr std::string_view kProgram = R"( +#[quickcheck(exhaustive)] +fn trivial(x: u5, y: u6) -> bool { !(x == u5::MAX && y == u6::MAX) } +)"; + constexpr const char* kModuleName = "test"; + constexpr const char* kFilename = "test.x"; + RunComparator jit_comparator(CompareMode::kJit); + ParseAndTestOptions options; + options.run_comparator = &jit_comparator; + options.vfs_factory = [kProgram] { + return std::make_unique(kProgram, "test.x"); + }; + XLS_ASSERT_OK_AND_ASSIGN( + TestResultData result, + ParseAndTest(kProgram, kModuleName, kFilename, options)); + EXPECT_THAT(result, IsTestResult(TestResult::kSomeFailed, /*test_cases=*/1, + /*ran_count=*/0, /*failed_count=*/1)); +} + TEST_P(RunRoutinesTest, NoSeedStillQuickChecks) { constexpr const char* kProgram = R"( fn id(x: bool) -> bool { x } @@ -306,13 +373,13 @@ TEST(QuickcheckTest, QuickCheckBits) { } )"; int64_t seed = 0; - int64_t num_tests = 1000; + QuickCheckTestCases test_cases = QuickCheckTestCases::Counted(1000); XLS_ASSERT_OK_AND_ASSIGN(xls::Function * function, Parser::ParseFunction(ir_text, &package)); RunComparator jit_comparator(CompareMode::kJit); XLS_ASSERT_OK_AND_ASSIGN( auto quickcheck_info, - DoQuickCheck(function, kFakeIrName, &jit_comparator, seed, num_tests)); + DoQuickCheck(function, kFakeIrName, &jit_comparator, seed, test_cases)); std::vector results = quickcheck_info.results; // If a counter-example was found, the last result will be 0. EXPECT_EQ(results.back(), Value(UBits(0, 1))); @@ -330,13 +397,13 @@ TEST(QuickcheckTest, QuickCheckArray) { } )"; int64_t seed = 0; - int64_t num_tests = 1000; + QuickCheckTestCases test_cases = QuickCheckTestCases::Counted(1000); XLS_ASSERT_OK_AND_ASSIGN(xls::Function * function, Parser::ParseFunction(ir_text, &package)); RunComparator jit_comparator(CompareMode::kJit); XLS_ASSERT_OK_AND_ASSIGN( auto quickcheck_info, - DoQuickCheck(function, kFakeIrName, &jit_comparator, seed, num_tests)); + DoQuickCheck(function, kFakeIrName, &jit_comparator, seed, test_cases)); std::vector results = quickcheck_info.results; EXPECT_EQ(results.back(), Value(UBits(0, 1))); } @@ -351,13 +418,13 @@ TEST(QuickcheckTest, QuickCheckTuple) { } )"; int64_t seed = 0; - int64_t num_tests = 1000; + QuickCheckTestCases test_cases = QuickCheckTestCases::Counted(1000); XLS_ASSERT_OK_AND_ASSIGN(xls::Function * function, Parser::ParseFunction(ir_text, &package)); RunComparator jit_comparator(CompareMode::kJit); XLS_ASSERT_OK_AND_ASSIGN( auto quickcheck_info, - DoQuickCheck(function, kFakeIrName, &jit_comparator, seed, num_tests)); + DoQuickCheck(function, kFakeIrName, &jit_comparator, seed, test_cases)); std::vector results = quickcheck_info.results; EXPECT_EQ(results.back(), Value(UBits(0, 1))); } @@ -372,13 +439,13 @@ TEST(QuickcheckTest, NumTests) { } )"; int64_t seed = 0; - int64_t num_tests = 5050; + QuickCheckTestCases test_cases = QuickCheckTestCases::Counted(5050); XLS_ASSERT_OK_AND_ASSIGN(xls::Function * function, Parser::ParseFunction(ir_text, &package)); RunComparator jit_comparator(CompareMode::kJit); XLS_ASSERT_OK_AND_ASSIGN( auto quickcheck_info, - DoQuickCheck(function, kFakeIrName, &jit_comparator, seed, num_tests)); + DoQuickCheck(function, kFakeIrName, &jit_comparator, seed, test_cases)); std::vector> argsets = quickcheck_info.arg_sets; std::vector results = quickcheck_info.results; @@ -397,16 +464,16 @@ TEST(QuickcheckTest, Seeding) { } )"; int64_t seed = 12345; - int64_t num_tests = 1000; + QuickCheckTestCases test_cases = QuickCheckTestCases::Counted(1000); XLS_ASSERT_OK_AND_ASSIGN(xls::Function * function, Parser::ParseFunction(ir_text, &package)); RunComparator jit_comparator(CompareMode::kJit); XLS_ASSERT_OK_AND_ASSIGN( auto quickcheck_info1, - DoQuickCheck(function, kFakeIrName, &jit_comparator, seed, num_tests)); + DoQuickCheck(function, kFakeIrName, &jit_comparator, seed, test_cases)); XLS_ASSERT_OK_AND_ASSIGN( auto quickcheck_info2, - DoQuickCheck(function, kFakeIrName, &jit_comparator, seed, num_tests)); + DoQuickCheck(function, kFakeIrName, &jit_comparator, seed, test_cases)); const auto& [argsets1, results1] = quickcheck_info1; const auto& [argsets2, results2] = quickcheck_info2; diff --git a/xls/dslx/tests/errors/error_modules_test.py b/xls/dslx/tests/errors/error_modules_test.py index 0c2efd9c19..b1cd9372e1 100644 --- a/xls/dslx/tests/errors/error_modules_test.py +++ b/xls/dslx/tests/errors/error_modules_test.py @@ -90,7 +90,7 @@ def test_failing_test_output(self): ) self.assertRegex(lines[5], r'\[ SEED [\d ]{16} \]') self.assertEqual( - lines[6], '[ RUN QUICKCHECK ] always_false count: 1000' + lines[6], '[ RUN QUICKCHECK ] always_false cases: test_count=default=1000' ) self.assertEqual(lines[7], '[ FAILED ] always_false') self.assertEqual(lines[8], '[=======================] 1 quickcheck(s) ran.') diff --git a/xls/ir/bits.h b/xls/ir/bits.h index 44e1d8b090..704a79b379 100644 --- a/xls/ir/bits.h +++ b/xls/ir/bits.h @@ -89,6 +89,10 @@ class Bits { return Bits(std::move(bitmap)); } + static Bits FromBitmapView(BitmapView bitmap) { + return Bits(bitmap.ToBitmap()); + } + // Note: we flatten into the pushbuffer with the MSb pushed first. void FlattenTo(BitPushBuffer* buffer) const { for (int64_t i = 0; i < bit_count(); ++i) { diff --git a/xls/ir/value.cc b/xls/ir/value.cc index 42f56f40b7..b98f4ef77f 100644 --- a/xls/ir/value.cc +++ b/xls/ir/value.cc @@ -252,6 +252,28 @@ void Value::FlattenTo(BitPushBuffer* buffer) const { LOG(FATAL) << "Invalid value kind: " << ValueKindToString(kind_); } +absl::Status Value::PopulateFrom(BitmapView bitmap) { + switch (kind_) { + case ValueKind::kBits: + payload_ = Bits::FromBitmapView(bitmap); + return absl::OkStatus(); + case ValueKind::kTuple: + case ValueKind::kArray: { + int64_t bit_index = 0; + for (Value& element : std::get>(payload_)) { + XLS_RETURN_IF_ERROR(element.PopulateFrom( + bitmap.Slice(bit_index, element.GetFlatBitCount()))); + bit_index += element.GetFlatBitCount(); + } + return absl::OkStatus(); + } + case ValueKind::kToken: + case ValueKind::kInvalid: + return absl::InvalidArgumentError(absl::StrFormat( + "Cannot populate value of kind: %s", ValueKindToString(kind_))); + } +} + absl::StatusOr> Value::GetElements() const { if (!std::holds_alternative>(payload_)) { return absl::InvalidArgumentError("Value does not hold elements."); diff --git a/xls/ir/value.h b/xls/ir/value.h index 01e5658d7e..3ef743aa20 100644 --- a/xls/ir/value.h +++ b/xls/ir/value.h @@ -133,6 +133,7 @@ class Value { // Serializes the contents of this value as bits in the buffer. void FlattenTo(BitPushBuffer* buffer) const; + absl::Status PopulateFrom(BitmapView bitmap); ValueKind kind() const { return kind_; } bool IsTuple() const { return kind_ == ValueKind::kTuple; } diff --git a/xls/ir/value_test.cc b/xls/ir/value_test.cc index bcc6f0d2d0..061fa11ab9 100644 --- a/xls/ir/value_test.cc +++ b/xls/ir/value_test.cc @@ -17,13 +17,13 @@ #include #include -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "xls/common/fuzzing/fuzztest.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/types/span.h" +#include "gmock/gmock.h" #include "google/protobuf/text_format.h" +#include "gtest/gtest.h" +#include "xls/common/fuzzing/fuzztest.h" #include "xls/common/proto_test_utils.h" #include "xls/common/status/matchers.h" #include "xls/ir/bits.h" @@ -701,6 +701,33 @@ void ProtoValueRoundTripWorks(const ValueProto& v) { FUZZ_TEST(ValueProto, ProtoValueRoundTripWorks) .WithDomains(fuzztest::Arbitrary()); + +void RoundTripFlattenToPopulateFrom(const ValueProto& vp) { + absl::StatusOr v = Value::FromProto(vp); + if (!v.ok()) { + // Don't bother with invalid values. + return; + } + TypeManager type_manager; + Type* type = type_manager.GetTypeForValue(v.value()); + if (TypeHasToken(type)) { + // Token values cannot be serialized. + return; + } + + BitPushBuffer push_buffer; + v->FlattenTo(&push_buffer); + InlineBitmap bitmap = push_buffer.ToBitmap(); + + Value round_trip = ZeroOfType(type); + ASSERT_TRUE(round_trip.SameTypeAs(v.value())); + XLS_ASSERT_OK(round_trip.PopulateFrom(BitmapView(bitmap))); + EXPECT_EQ(v.value(), round_trip); +} + +FUZZ_TEST(ValueProto, RoundTripFlattenToPopulateFrom) + .WithDomains(fuzztest::Arbitrary()); + } // namespace } // namespace xls