From 789ec6699b73dc1e80248e26134b57b51de8de33 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Thu, 21 Mar 2024 10:29:00 -0700 Subject: [PATCH 1/3] storage: remove unused header Signed-off-by: Noah Watkins --- src/v/storage/log_manager.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/v/storage/log_manager.h b/src/v/storage/log_manager.h index 8c485e138d4a..b49667521a89 100644 --- a/src/v/storage/log_manager.h +++ b/src/v/storage/log_manager.h @@ -13,7 +13,6 @@ #include "base/seastarx.h" #include "base/units.h" -#include "cluster/topic_table.h" #include "config/property.h" #include "container/intrusive_list_helpers.h" #include "features/feature_table.h" From db2995b95b153402b20db195f5a62a1e3eb5cd43 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 23 Mar 2024 10:06:04 -0700 Subject: [PATCH 2/3] storage: remove unused headers Signed-off-by: Noah Watkins --- src/v/storage/log_manager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v/storage/log_manager.cc b/src/v/storage/log_manager.cc index bdb89b66ee31..56637970a736 100644 --- a/src/v/storage/log_manager.cc +++ b/src/v/storage/log_manager.cc @@ -11,8 +11,6 @@ #include "base/likely.h" #include "base/vlog.h" -#include "cluster/cluster_utils.h" -#include "cluster/topic_table.h" #include "config/configuration.h" #include "model/fundamental.h" #include "model/metadata.h" @@ -66,6 +64,8 @@ #include #include +using namespace std::chrono_literals; + namespace storage { using logs_type = absl::flat_hash_map; From 623cdc5659a5d4238420d8c7408d7c95a12a56f3 Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 23 Mar 2024 10:23:07 -0700 Subject: [PATCH 3/3] tests: remove module-specific bits from test_utils Test utils is a global library, but had lots of bits from modules (e.g. security, model, etc...). This resulted in a lot of implicit dependencies. This commit splits it and includes the specializations in each module. Signed-off-by: Noah Watkins --- src/v/cluster/tests/randoms.h | 49 +++++ src/v/cluster/tests/serialization_rt_test.cc | 1 + src/v/compat/acls_generator.h | 1 + src/v/model/tests/randoms.h | 32 +++ src/v/net/tests/randoms.h | 25 +++ src/v/raft/persisted_stm.cc | 1 + .../tests/configuration_serialization_test.cc | 1 + .../tests/ephemeral_credential_store_test.cc | 1 + src/v/security/tests/randoms.h | 130 ++++++++++++ src/v/storage/offset_translator.cc | 1 + .../tests/compaction_index_format_tests.cc | 1 + src/v/test_utils/randoms.h | 195 +----------------- src/v/transform/tests/test_fixture.cc | 3 + src/v/wasm/tests/wasm_fixture.cc | 3 + src/v/wasm/tests/wasm_transform_bench.cc | 3 + 15 files changed, 254 insertions(+), 193 deletions(-) create mode 100644 src/v/net/tests/randoms.h create mode 100644 src/v/security/tests/randoms.h diff --git a/src/v/cluster/tests/randoms.h b/src/v/cluster/tests/randoms.h index e5ebe7cfebb9..4cd7ee9b9a75 100644 --- a/src/v/cluster/tests/randoms.h +++ b/src/v/cluster/tests/randoms.h @@ -12,6 +12,8 @@ #pragma once #include "cluster/health_monitor_types.h" +#include "cluster/partition_balancer_state.h" +#include "cluster/partition_balancer_types.h" #include "cluster/tx_snapshot_utils.h" #include "model/tests/randoms.h" #include "random/generators.h" @@ -193,3 +195,50 @@ inline tx_snapshot_v3::tx_seqs_snapshot random_tx_seqs_snapshot() { } } // namespace cluster + +namespace tests { + +inline cluster::producer_ptr +random_producer_state(cluster::producer_state_manager& psm) { + return ss::make_lw_shared( + psm, + model::producer_identity{ + random_generators::get_int(), + random_generators::get_int()}, + tests::random_named_int(), + ss::noncopyable_function{}); +} + +inline cluster::partition_balancer_status random_balancer_status() { + return random_generators::random_choice({ + cluster::partition_balancer_status::off, + cluster::partition_balancer_status::starting, + cluster::partition_balancer_status::ready, + cluster::partition_balancer_status::in_progress, + cluster::partition_balancer_status::stalled, + }); +} + +inline cluster::partition_balancer_violations::unavailable_node +random_unavailable_node() { + return { + tests::random_named_int(), + model::timestamp(random_generators::get_int())}; +} + +inline cluster::partition_balancer_violations::full_node random_full_node() { + return { + tests::random_named_int(), + random_generators::get_int()}; +} + +inline cluster::partition_balancer_violations +random_partition_balancer_violations() { + auto random_un_gen = tests::random_vector( + []() { return random_unavailable_node(); }); + auto random_fn_gen = tests::random_vector( + []() { return random_full_node(); }); + return {std::move(random_un_gen), std::move(random_fn_gen)}; +} + +} // namespace tests diff --git a/src/v/cluster/tests/serialization_rt_test.cc b/src/v/cluster/tests/serialization_rt_test.cc index 139125fd2918..1a90d9f8ea08 100644 --- a/src/v/cluster/tests/serialization_rt_test.cc +++ b/src/v/cluster/tests/serialization_rt_test.cc @@ -28,6 +28,7 @@ #include "random/generators.h" #include "reflection/adl.h" #include "reflection/async_adl.h" +#include "security/tests/randoms.h" #include "storage/types.h" #include "test_utils/randoms.h" #include "test_utils/rpc.h" diff --git a/src/v/compat/acls_generator.h b/src/v/compat/acls_generator.h index 45234824f61b..18eb512d7542 100644 --- a/src/v/compat/acls_generator.h +++ b/src/v/compat/acls_generator.h @@ -14,6 +14,7 @@ #include "cluster/types.h" #include "compat/cluster_generator.h" #include "compat/generator.h" +#include "security/tests/randoms.h" #include "test_utils/randoms.h" namespace compat { diff --git a/src/v/model/tests/randoms.h b/src/v/model/tests/randoms.h index ee74ee764437..7a0d57460772 100644 --- a/src/v/model/tests/randoms.h +++ b/src/v/model/tests/randoms.h @@ -18,6 +18,7 @@ #include "model/metadata.h" #include "model/record.h" #include "model/timestamp.h" +#include "net/tests/randoms.h" #include "pandaproxy/schema_registry/subject_name_strategy.h" #include "random/generators.h" #include "test_utils/randoms.h" @@ -191,3 +192,34 @@ random_subject_name_strategy() { } } // namespace model + +namespace tests { + +inline model::record_batch_type random_batch_type() { + return random_generators::random_choice( + std::vector{ + model::record_batch_type::raft_data, + model::record_batch_type::raft_configuration, + model::record_batch_type::controller, + model::record_batch_type::kvstore, + model::record_batch_type::checkpoint, + model::record_batch_type::topic_management_cmd, + model::record_batch_type::ghost_batch, + model::record_batch_type::id_allocator, + model::record_batch_type::tx_prepare, + model::record_batch_type::tx_fence, + model::record_batch_type::tm_update, + model::record_batch_type::user_management_cmd, + model::record_batch_type::acl_management_cmd, + model::record_batch_type::group_prepare_tx, + model::record_batch_type::group_commit_tx, + model::record_batch_type::group_abort_tx, + model::record_batch_type::node_management_cmd, + model::record_batch_type::data_policy_management_cmd, + model::record_batch_type::archival_metadata, + model::record_batch_type::cluster_config_cmd, + model::record_batch_type::feature_update, + }); +} + +} // namespace tests diff --git a/src/v/net/tests/randoms.h b/src/v/net/tests/randoms.h new file mode 100644 index 000000000000..40e903643b1b --- /dev/null +++ b/src/v/net/tests/randoms.h @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Redpanda Data, Inc. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.md + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0 + */ +#pragma once + +#include "net/unresolved_address.h" +#include "random/generators.h" + +namespace tests { + +inline net::unresolved_address random_net_address() { + return net::unresolved_address( + random_generators::gen_alphanum_string( + random_generators::get_int(1, 100)), + random_generators::get_int(1025, 65535)); +} + +} // namespace tests diff --git a/src/v/raft/persisted_stm.cc b/src/v/raft/persisted_stm.cc index 44629c146b15..1a42dbb53f2e 100644 --- a/src/v/raft/persisted_stm.cc +++ b/src/v/raft/persisted_stm.cc @@ -10,6 +10,7 @@ #include "raft/persisted_stm.h" #include "bytes/iostream.h" +#include "cluster/types.h" #include "raft/consensus.h" #include "raft/errc.h" #include "raft/offset_monitor.h" diff --git a/src/v/raft/tests/configuration_serialization_test.cc b/src/v/raft/tests/configuration_serialization_test.cc index 58fb3561b910..96ca85dcf60f 100644 --- a/src/v/raft/tests/configuration_serialization_test.cc +++ b/src/v/raft/tests/configuration_serialization_test.cc @@ -8,6 +8,7 @@ // by the Apache License, Version 2.0 #include "bytes/iobuf_parser.h" +#include "model/adl_serde.h" #include "model/metadata.h" #include "model/tests/random_batch.h" #include "model/tests/randoms.h" diff --git a/src/v/security/tests/ephemeral_credential_store_test.cc b/src/v/security/tests/ephemeral_credential_store_test.cc index 6cf82e45b74e..87bb31a271d3 100644 --- a/src/v/security/tests/ephemeral_credential_store_test.cc +++ b/src/v/security/tests/ephemeral_credential_store_test.cc @@ -12,6 +12,7 @@ #include "security/ephemeral_credential_store.h" #include "security/scram_authenticator.h" #include "security/scram_credential.h" +#include "security/tests/randoms.h" #include "security/types.h" #include "test_utils/randoms.h" diff --git a/src/v/security/tests/randoms.h b/src/v/security/tests/randoms.h new file mode 100644 index 000000000000..2351d39ba200 --- /dev/null +++ b/src/v/security/tests/randoms.h @@ -0,0 +1,130 @@ +/* + * Copyright 2020 Redpanda Data, Inc. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.md + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0 + */ +#pragma once + +#include "bytes/random.h" +#include "random/generators.h" +#include "security/acl.h" +#include "security/scram_credential.h" +#include "test_utils/randoms.h" + +namespace tests { + +inline security::scram_credential random_credential() { + return security::scram_credential( + random_generators::get_bytes(256), + random_generators::get_bytes(256), + random_generators::get_bytes(256), + random_generators::get_int(1, 10)); +} + +inline security::resource_type random_resource_type() { + return random_generators::random_choice( + {security::resource_type::cluster, + security::resource_type::group, + security::resource_type::topic, + security::resource_type::transactional_id}); +} + +inline security::pattern_type random_pattern_type() { + return random_generators::random_choice( + {security::pattern_type::literal, security::pattern_type::prefixed}); +} + +inline security::resource_pattern random_resource_pattern() { + return { + random_resource_type(), + random_generators::gen_alphanum_string(10), + random_pattern_type()}; +} + +inline security::acl_principal random_acl_principal() { + return { + security::principal_type::user, + random_generators::gen_alphanum_string(12)}; +} + +inline security::acl_host create_acl_host() { + return security::acl_host(ss::net::inet_address("127.0.0.1")); +} + +inline security::acl_operation random_acl_operation() { + return random_generators::random_choice( + {security::acl_operation::all, + security::acl_operation::alter, + security::acl_operation::alter_configs, + security::acl_operation::describe_configs, + security::acl_operation::cluster_action, + security::acl_operation::create, + security::acl_operation::remove, + security::acl_operation::read, + security::acl_operation::idempotent_write, + security::acl_operation::describe}); +} + +inline security::acl_permission random_acl_permission() { + return random_generators::random_choice( + {security::acl_permission::allow, security::acl_permission::deny}); +} + +inline security::acl_entry random_acl_entry() { + return { + random_acl_principal(), + create_acl_host(), + random_acl_operation(), + random_acl_permission()}; +} + +inline security::acl_binding random_acl_binding() { + return {random_resource_pattern(), random_acl_entry()}; +} + +inline security::resource_pattern_filter random_resource_pattern_filter() { + auto resource = tests::random_optional( + [] { return random_resource_type(); }); + + auto name = tests::random_optional( + [] { return random_generators::gen_alphanum_string(14); }); + + auto pattern = tests::random_optional([] { + using ret_t = std::variant< + security::pattern_type, + security::resource_pattern_filter::pattern_match>; + if (tests::random_bool()) { + return ret_t(random_pattern_type()); + } else { + return ret_t(security::resource_pattern_filter::pattern_match{}); + } + }); + + return {resource, std::move(name), pattern}; +} + +inline security::acl_entry_filter random_acl_entry_filter() { + auto principal = tests::random_optional( + [] { return random_acl_principal(); }); + + auto host = tests::random_optional([] { return create_acl_host(); }); + + auto operation = tests::random_optional( + [] { return random_acl_operation(); }); + + auto permission = tests::random_optional( + [] { return random_acl_permission(); }); + + return {std::move(principal), host, operation, permission}; +} + +inline security::acl_binding_filter random_acl_binding_filter() { + return {random_resource_pattern_filter(), random_acl_entry_filter()}; +} + +} // namespace tests diff --git a/src/v/storage/offset_translator.cc b/src/v/storage/offset_translator.cc index ac7189ded094..e39ae5f3fd0b 100644 --- a/src/v/storage/offset_translator.cc +++ b/src/v/storage/offset_translator.cc @@ -12,6 +12,7 @@ #include "storage/offset_translator.h" #include "base/vlog.h" +#include "reflection/adl.h" #include "storage/api.h" #include "storage/kvstore.h" #include "storage/logger.h" diff --git a/src/v/storage/tests/compaction_index_format_tests.cc b/src/v/storage/tests/compaction_index_format_tests.cc index 7dca4a41e830..2f81296effd7 100644 --- a/src/v/storage/tests/compaction_index_format_tests.cc +++ b/src/v/storage/tests/compaction_index_format_tests.cc @@ -11,6 +11,7 @@ #include "bytes/bytes.h" #include "bytes/iobuf_parser.h" #include "bytes/random.h" +#include "model/tests/randoms.h" #include "random/generators.h" #include "reflection/adl.h" #include "storage/compacted_index.h" diff --git a/src/v/test_utils/randoms.h b/src/v/test_utils/randoms.h index af34ddcc2575..f9aa21400a30 100644 --- a/src/v/test_utils/randoms.h +++ b/src/v/test_utils/randoms.h @@ -8,17 +8,12 @@ * the Business Source License, use of this software will be governed * by the Apache License, Version 2.0 */ - #pragma once + #include "bytes/random.h" -#include "cluster/partition_balancer_types.h" -#include "cluster/producer_state.h" #include "container/fragmented_vector.h" -#include "model/metadata.h" -#include "net/unresolved_address.h" #include "random/generators.h" -#include "security/acl.h" -#include "security/scram_credential.h" +#include "tristate.h" #include #include @@ -38,13 +33,6 @@ namespace tests { -inline net::unresolved_address random_net_address() { - return net::unresolved_address( - random_generators::gen_alphanum_string( - random_generators::get_int(1, 100)), - random_generators::get_int(1025, 65535)); -} - inline bool random_bool() { return random_generators::get_int(0, 100) > 50; } template @@ -203,17 +191,6 @@ inline absl::btree_set random_btree_set(Fn&& gen, size_t size = 20) { return random_set(std::forward(gen), size); } -inline cluster::producer_ptr -random_producer_state(cluster::producer_state_manager& psm) { - return ss::make_lw_shared( - psm, - model::producer_identity{ - random_generators::get_int(), - random_generators::get_int()}, - random_named_int(), - ss::noncopyable_function{}); -} - /* * Generate a random duration. Notice that random value is multiplied by 10^6. * This is so that roundtrip from ns->ms->ns will work as expected. @@ -224,172 +201,4 @@ inline Dur random_duration() { random_generators::get_int(-100000, 100000) * 1000000); } -inline cluster::partition_balancer_status random_balancer_status() { - return random_generators::random_choice({ - cluster::partition_balancer_status::off, - cluster::partition_balancer_status::starting, - cluster::partition_balancer_status::ready, - cluster::partition_balancer_status::in_progress, - cluster::partition_balancer_status::stalled, - }); -} - -inline cluster::partition_balancer_violations::unavailable_node -random_unavailable_node() { - return { - tests::random_named_int(), - model::timestamp(random_generators::get_int())}; -} - -inline cluster::partition_balancer_violations::full_node random_full_node() { - return { - tests::random_named_int(), - random_generators::get_int()}; -} - -inline cluster::partition_balancer_violations -random_partition_balancer_violations() { - auto random_un_gen = tests::random_vector( - []() { return random_unavailable_node(); }); - auto random_fn_gen = tests::random_vector( - []() { return random_full_node(); }); - return {std::move(random_un_gen), std::move(random_fn_gen)}; -} - -inline security::scram_credential random_credential() { - return security::scram_credential( - random_generators::get_bytes(256), - random_generators::get_bytes(256), - random_generators::get_bytes(256), - random_generators::get_int(1, 10)); -} - -inline security::resource_type random_resource_type() { - return random_generators::random_choice( - {security::resource_type::cluster, - security::resource_type::group, - security::resource_type::topic, - security::resource_type::transactional_id}); -} - -inline security::pattern_type random_pattern_type() { - return random_generators::random_choice( - {security::pattern_type::literal, security::pattern_type::prefixed}); -} - -inline security::resource_pattern random_resource_pattern() { - return { - random_resource_type(), - random_generators::gen_alphanum_string(10), - random_pattern_type()}; -} - -inline security::acl_principal random_acl_principal() { - return { - security::principal_type::user, - random_generators::gen_alphanum_string(12)}; -} - -inline security::acl_host create_acl_host() { - return security::acl_host(ss::net::inet_address("127.0.0.1")); -} - -inline security::acl_operation random_acl_operation() { - return random_generators::random_choice( - {security::acl_operation::all, - security::acl_operation::alter, - security::acl_operation::alter_configs, - security::acl_operation::describe_configs, - security::acl_operation::cluster_action, - security::acl_operation::create, - security::acl_operation::remove, - security::acl_operation::read, - security::acl_operation::idempotent_write, - security::acl_operation::describe}); -} - -inline model::record_batch_type random_batch_type() { - return random_generators::random_choice( - std::vector{ - model::record_batch_type::raft_data, - model::record_batch_type::raft_configuration, - model::record_batch_type::controller, - model::record_batch_type::kvstore, - model::record_batch_type::checkpoint, - model::record_batch_type::topic_management_cmd, - model::record_batch_type::ghost_batch, - model::record_batch_type::id_allocator, - model::record_batch_type::tx_prepare, - model::record_batch_type::tx_fence, - model::record_batch_type::tm_update, - model::record_batch_type::user_management_cmd, - model::record_batch_type::acl_management_cmd, - model::record_batch_type::group_prepare_tx, - model::record_batch_type::group_commit_tx, - model::record_batch_type::group_abort_tx, - model::record_batch_type::node_management_cmd, - model::record_batch_type::data_policy_management_cmd, - model::record_batch_type::archival_metadata, - model::record_batch_type::cluster_config_cmd, - model::record_batch_type::feature_update, - }); -} - -inline security::acl_permission random_acl_permission() { - return random_generators::random_choice( - {security::acl_permission::allow, security::acl_permission::deny}); -} - -inline security::acl_entry random_acl_entry() { - return { - random_acl_principal(), - create_acl_host(), - random_acl_operation(), - random_acl_permission()}; -} - -inline security::acl_binding random_acl_binding() { - return {random_resource_pattern(), random_acl_entry()}; -} - -inline security::resource_pattern_filter random_resource_pattern_filter() { - auto resource = tests::random_optional( - [] { return random_resource_type(); }); - - auto name = tests::random_optional( - [] { return random_generators::gen_alphanum_string(14); }); - - auto pattern = tests::random_optional([] { - using ret_t = std::variant< - security::pattern_type, - security::resource_pattern_filter::pattern_match>; - if (tests::random_bool()) { - return ret_t(random_pattern_type()); - } else { - return ret_t(security::resource_pattern_filter::pattern_match{}); - } - }); - - return {resource, std::move(name), pattern}; -} - -inline security::acl_entry_filter random_acl_entry_filter() { - auto principal = tests::random_optional( - [] { return random_acl_principal(); }); - - auto host = tests::random_optional([] { return create_acl_host(); }); - - auto operation = tests::random_optional( - [] { return random_acl_operation(); }); - - auto permission = tests::random_optional( - [] { return random_acl_permission(); }); - - return {std::move(principal), host, operation, permission}; -} - -inline security::acl_binding_filter random_acl_binding_filter() { - return {random_resource_pattern_filter(), random_acl_entry_filter()}; -} - } // namespace tests diff --git a/src/v/transform/tests/test_fixture.cc b/src/v/transform/tests/test_fixture.cc index b7cc46e1470e..eaa4b9d5f55d 100644 --- a/src/v/transform/tests/test_fixture.cc +++ b/src/v/transform/tests/test_fixture.cc @@ -23,11 +23,14 @@ #include +#include #include #include #include #include +using namespace std::chrono_literals; + namespace transform::testing { ss::future<> fake_sink::write(ss::chunked_fifo batches) { co_await _cork.wait(); diff --git a/src/v/wasm/tests/wasm_fixture.cc b/src/v/wasm/tests/wasm_fixture.cc index 45879e89a80f..6ff23f74d90d 100644 --- a/src/v/wasm/tests/wasm_fixture.cc +++ b/src/v/wasm/tests/wasm_fixture.cc @@ -32,9 +32,12 @@ #include +#include #include #include +using namespace std::chrono_literals; + namespace { // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables,cert-err58-cpp) static ss::logger dummy_logger("wasm_test_logger"); diff --git a/src/v/wasm/tests/wasm_transform_bench.cc b/src/v/wasm/tests/wasm_transform_bench.cc index 583ea610c078..02eab68ede7b 100644 --- a/src/v/wasm/tests/wasm_transform_bench.cc +++ b/src/v/wasm/tests/wasm_transform_bench.cc @@ -28,9 +28,12 @@ #include #include +#include #include #include +using namespace std::chrono_literals; + template class WasmBenchTest { public: