Skip to content
This repository has been archived by the owner on Feb 20, 2023. It is now read-only.

Commit

Permalink
Synchronous Replication (#1472)
Browse files Browse the repository at this point in the history 
Co-authored-by: tpan496 <tianleip@andrew.cmu.edu>
  • Loading branch information
@lmwnshn @tpan496
lmwnshn and tpan496 authored Mar 9, 2021
1 parent 6d7da24 commit 98e877b
Show file tree
Hide file tree
Showing 74 changed files with 2,091 additions and 755 deletions.
14 changes: 13 additions & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
# HEADER Dependencies.
# HEADER noisepage libraries.
# HEADER noisepage binary.
# HEADER noisepage miscellaneous files needed for operation.
# HEADER util_static and util_shared libraries.
# HEADER hack_bytecode_handlers_ir target.
# HEADER gen_opt_bc binary.
Expand Down Expand Up @@ -697,6 +698,18 @@ target_compile_options(noisepage PRIVATE "-Werror" "-Wall")
target_link_libraries(noisepage noisepage_static)
set_target_properties(noisepage PROPERTIES CXX_EXTENSIONS OFF ENABLE_EXPORTS ON)

#######################################################################################################################
# HEADER noisepage miscellaneous files needed for operation.
# replication.config : The identities, hostnames, and ports, of all primaries and replicas.
#######################################################################################################################

# replication.config specifies the identities, hostnames, and ports, and is needed when replication is enabled.
add_custom_command(
TARGET noisepage
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy "${BUILD_SUPPORT_DIR}/data/replication.config" "${CMAKE_BINARY_DIR}/bin/replication.config"
)

#######################################################################################################################
# HEADER util_static and util_shared libraries.
# util_static : table_generator and table_reader for tests and benchmarks.
Expand Down Expand Up @@ -781,7 +794,6 @@ add_dependencies(noisepage_static bitcode_generator)
#######################################################################################################################
# HEADER tpl binary.
# tpl : Standalone binary for testing .tpl files and commands.
# Also generates POST_BUILD bytecode_handlers_ir.bc.
#######################################################################################################################

add_executable(tpl util/execution/tpl.cpp)
Expand Down
4 changes: 4 additions & 0 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ pipeline {
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15723', label: 'Kill PID(15723)'
sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
sh script: 'cd build/bin && PYTHONPATH=../.. timeout 20m python3 -m script.testing.replication.tests_simple --build-type=debug', label: 'Replication (Simple)'
sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=simple', label: 'UnitTest (Simple)'
sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=extended', label: 'UnitTest (Extended)'
sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=extended -a "pipeline_metrics_enable=True" -a "pipeline_metrics_sample_rate=100" -a "counters_enable=True" -a "query_trace_metrics_enable=True"', label: 'UnitTest (Extended with pipeline metrics, counters, and query trace metrics)'
Expand Down Expand Up @@ -235,6 +236,7 @@ pipeline {
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15723', label: 'Kill PID(15723)'
sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
sh script: 'cd build/bin && PYTHONPATH=../.. timeout 20m python3 -m script.testing.replication.tests_simple --build-type=debug', label: 'Replication (Simple)'
sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=simple', label: 'UnitTest (Simple)'
sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=extended', label: 'UnitTest (Extended)'
sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
Expand Down Expand Up @@ -273,6 +275,7 @@ pipeline {
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15723', label: 'Kill PID(15723)'
sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
sh script: 'cd build/bin && PYTHONPATH=../.. timeout 20m python3 -m script.testing.replication.tests_simple --build-type=release', label: 'Replication (Simple)'
sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=release --query-mode=simple', label: 'UnitTest (Simple)'
sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=release --query-mode=extended', label: 'UnitTest (Extended)'
sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
Expand Down Expand Up @@ -315,6 +318,7 @@ pipeline {
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15723', label: 'Kill PID(15723)'
sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
sh script: 'cd build/bin && PYTHONPATH=../.. timeout 20m python3 -m script.testing.replication.tests_simple --build-type=release', label: 'Replication (Simple)'
sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=release --query-mode=simple', label: 'UnitTest (Simple)'
sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=release --query-mode=extended', label: 'UnitTest (Extended)'
sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
Expand Down
17 changes: 9 additions & 8 deletions benchmark/integration/tpcc_benchmark.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ class TPCCBenchmark : public benchmark::Fixture {
storage::RecordBufferSegmentPool buffer_pool_{buffersegment_size_limit_, buffersegment_reuse_limit_};
std::default_random_engine generator_;
storage::LogManager *log_manager_ = DISABLED; // logging enabled will override this value
common::ConcurrentBlockingQueue<storage::BufferedLogWriter *> empty_buffer_queue_;

// Settings for log manager
const uint64_t num_log_buffers_ = 100;
Expand Down Expand Up @@ -173,10 +174,10 @@ BENCHMARK_DEFINE_F(TPCCBenchmark, ScaleFactor4WithLogging)(benchmark::State &sta
unlink(noisepage::BenchmarkConfig::logfile_path.data());
thread_registry_ = new common::DedicatedThreadRegistry(DISABLED);
// we need transactions, TPCC database, and GC
log_manager_ =
new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
common::ManagedPointer(&buffer_pool_), common::ManagedPointer(thread_registry_));
log_manager_ = new storage::LogManager(
noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_, log_serialization_interval_,
log_persist_interval_, log_persist_threshold_, common::ManagedPointer(&buffer_pool_),
common::ManagedPointer(&empty_buffer_queue_), DISABLED, common::ManagedPointer(thread_registry_));
log_manager_->Start();
transaction::TimestampManager timestamp_manager;
transaction::DeferredActionManager deferred_action_manager{common::ManagedPointer(&timestamp_manager)};
Expand Down Expand Up @@ -275,10 +276,10 @@ BENCHMARK_DEFINE_F(TPCCBenchmark, ScaleFactor4WithLoggingAndMetrics)(benchmark::
metrics_manager->EnableMetric(metrics::MetricsComponent::LOGGING);
thread_registry_ = new common::DedicatedThreadRegistry{common::ManagedPointer(metrics_manager)};
// we need transactions, TPCC database, and GC
log_manager_ =
new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
common::ManagedPointer(&buffer_pool_), common::ManagedPointer(thread_registry_));
log_manager_ = new storage::LogManager(
noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_, log_serialization_interval_,
log_persist_interval_, log_persist_threshold_, common::ManagedPointer(&buffer_pool_),
common::ManagedPointer(&empty_buffer_queue_), DISABLED, common::ManagedPointer(thread_registry_));
log_manager_->Start();
transaction::TimestampManager timestamp_manager;
transaction::DeferredActionManager deferred_action_manager{common::ManagedPointer(&timestamp_manager)};
Expand Down
46 changes: 26 additions & 20 deletions benchmark/metrics/logging_metrics_benchmark.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ class LoggingMetricsBenchmark : public benchmark::Fixture {
const std::chrono::microseconds gc_period_{1000};
const std::chrono::microseconds metrics_period_{10000};
common::DedicatedThreadRegistry *thread_registry_ = nullptr;
common::ConcurrentBlockingQueue<storage::BufferedLogWriter *> empty_buffer_queue_;

// Settings for log manager
const uint64_t num_log_buffers_ = 100;
Expand All @@ -55,10 +56,11 @@ BENCHMARK_DEFINE_F(LoggingMetricsBenchmark, TPCCish)(benchmark::State &state) {
metrics_manager->EnableMetric(metrics::MetricsComponent::LOGGING);
thread_registry_ = new common::DedicatedThreadRegistry(common::ManagedPointer(metrics_manager));

log_manager_ = new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
common::ManagedPointer(&buffer_pool_),
common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
log_manager_ =
new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
common::ManagedPointer(&buffer_pool_), common::ManagedPointer(&empty_buffer_queue_),
DISABLED, common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
log_manager_->Start();
LargeDataTableBenchmarkObject tested(attr_sizes_, initial_table_size_, txn_length, insert_update_select_ratio,
&block_store_, &buffer_pool_, &generator_, true, log_manager_);
Expand Down Expand Up @@ -106,10 +108,11 @@ BENCHMARK_DEFINE_F(LoggingMetricsBenchmark, HighAbortRate)(benchmark::State &sta
metrics_manager->EnableMetric(metrics::MetricsComponent::LOGGING);
thread_registry_ = new common::DedicatedThreadRegistry(common::ManagedPointer(metrics_manager));

log_manager_ = new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
common::ManagedPointer(&buffer_pool_),
common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
log_manager_ =
new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
common::ManagedPointer(&buffer_pool_), common::ManagedPointer(&empty_buffer_queue_),
DISABLED, common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
log_manager_->Start();
LargeDataTableBenchmarkObject tested(attr_sizes_, 1000, txn_length, insert_update_select_ratio, &block_store_,
&buffer_pool_, &generator_, true, log_manager_);
Expand Down Expand Up @@ -157,10 +160,11 @@ BENCHMARK_DEFINE_F(LoggingMetricsBenchmark, SingleStatementInsert)(benchmark::St
metrics_manager->EnableMetric(metrics::MetricsComponent::LOGGING);
thread_registry_ = new common::DedicatedThreadRegistry(common::ManagedPointer(metrics_manager));

log_manager_ = new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
common::ManagedPointer(&buffer_pool_),
common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
log_manager_ =
new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
common::ManagedPointer(&buffer_pool_), common::ManagedPointer(&empty_buffer_queue_),
DISABLED, common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
log_manager_->Start();
LargeDataTableBenchmarkObject tested(attr_sizes_, 0, txn_length, insert_update_select_ratio, &block_store_,
&buffer_pool_, &generator_, true, log_manager_);
Expand Down Expand Up @@ -208,10 +212,11 @@ BENCHMARK_DEFINE_F(LoggingMetricsBenchmark, SingleStatementUpdate)(benchmark::St
metrics_manager->EnableMetric(metrics::MetricsComponent::LOGGING);
thread_registry_ = new common::DedicatedThreadRegistry(common::ManagedPointer(metrics_manager));

log_manager_ = new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
common::ManagedPointer(&buffer_pool_),
common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
log_manager_ =
new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
common::ManagedPointer(&buffer_pool_), common::ManagedPointer(&empty_buffer_queue_),
DISABLED, common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
log_manager_->Start();
LargeDataTableBenchmarkObject tested(attr_sizes_, initial_table_size_, txn_length, insert_update_select_ratio,
&block_store_, &buffer_pool_, &generator_, true, log_manager_);
Expand Down Expand Up @@ -259,10 +264,11 @@ BENCHMARK_DEFINE_F(LoggingMetricsBenchmark, SingleStatementSelect)(benchmark::St
metrics_manager->EnableMetric(metrics::MetricsComponent::LOGGING);
thread_registry_ = new common::DedicatedThreadRegistry(common::ManagedPointer(metrics_manager));

log_manager_ = new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
common::ManagedPointer(&buffer_pool_),
common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
log_manager_ =
new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
common::ManagedPointer(&buffer_pool_), common::ManagedPointer(&empty_buffer_queue_),
DISABLED, common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
log_manager_->Start();
LargeDataTableBenchmarkObject tested(attr_sizes_, initial_table_size_, txn_length, insert_update_select_ratio,
&block_store_, &buffer_pool_, &generator_, true, log_manager_);
Expand Down
Loading

0 comments on commit 98e877b

Please sign in to comment.