Synchronous Replication

CMakeLists.txt

@@ -53,6 +53,7 @@
 #   HEADER Dependencies.
 #   HEADER noisepage libraries.
 #   HEADER noisepage binary.
+#   HEADER noisepage miscellaneous files needed for operation.
 #   HEADER util_static and util_shared libraries.
 #   HEADER hack_bytecode_handlers_ir target.
 #   HEADER gen_opt_bc binary.
@@ -697,6 +698,18 @@ target_compile_options(noisepage PRIVATE "-Werror" "-Wall")
 target_link_libraries(noisepage noisepage_static)
 set_target_properties(noisepage PROPERTIES CXX_EXTENSIONS OFF ENABLE_EXPORTS ON)
 
+#######################################################################################################################
+# HEADER noisepage miscellaneous files needed for operation.
+# replication.config    :   The identities, hostnames, and ports, of all primaries and replicas.
+#######################################################################################################################
+
+# replication.config specifies the identities, hostnames, and ports, and is needed when replication is enabled.
+add_custom_command(
+        TARGET noisepage
+        POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy "${BUILD_SUPPORT_DIR}/data/replication.config" "${CMAKE_BINARY_DIR}/bin/replication.config"
+        )
+
 #######################################################################################################################
 # HEADER util_static and util_shared libraries.
 # util_static           :   table_generator and table_reader for tests and benchmarks.
@@ -781,7 +794,6 @@ add_dependencies(noisepage_static bitcode_generator)
 #######################################################################################################################
 # HEADER tpl binary.
 # tpl                   :   Standalone binary for testing .tpl files and commands.
-#                           Also generates POST_BUILD bytecode_handlers_ir.bc.
 #######################################################################################################################
 
 add_executable(tpl util/execution/tpl.cpp)

Jenkinsfile

@@ -138,6 +138,7 @@ pipeline {
                         sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
                         sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15723', label: 'Kill PID(15723)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
+                        sh script: 'cd build/bin && PYTHONPATH=../.. timeout 20m python3 -m script.testing.replication.tests_simple --build-type=debug', label: 'Replication (Simple)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=simple', label: 'UnitTest (Simple)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=extended', label: 'UnitTest (Extended)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=extended -a "pipeline_metrics_enable=True" -a "pipeline_metrics_sample_rate=100" -a "counters_enable=True" -a "query_trace_metrics_enable=True"', label: 'UnitTest (Extended with pipeline metrics, counters, and query trace metrics)'
@@ -235,6 +236,7 @@ pipeline {
                         sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
                         sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15723', label: 'Kill PID(15723)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
+                        sh script: 'cd build/bin && PYTHONPATH=../.. timeout 20m python3 -m script.testing.replication.tests_simple --build-type=debug', label: 'Replication (Simple)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=simple', label: 'UnitTest (Simple)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=extended', label: 'UnitTest (Extended)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
@@ -273,6 +275,7 @@ pipeline {
                         sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
                         sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15723', label: 'Kill PID(15723)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
+                        sh script: 'cd build/bin && PYTHONPATH=../.. timeout 20m python3 -m script.testing.replication.tests_simple --build-type=release', label: 'Replication (Simple)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=release --query-mode=simple', label: 'UnitTest (Simple)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=release --query-mode=extended', label: 'UnitTest (Extended)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
@@ -315,6 +318,7 @@ pipeline {
                         sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
                         sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15723', label: 'Kill PID(15723)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
+                        sh script: 'cd build/bin && PYTHONPATH=../.. timeout 20m python3 -m script.testing.replication.tests_simple --build-type=release', label: 'Replication (Simple)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=release --query-mode=simple', label: 'UnitTest (Simple)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=release --query-mode=extended', label: 'UnitTest (Extended)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'

benchmark/integration/tpcc_benchmark.cpp

@@ -40,6 +40,7 @@ class TPCCBenchmark : public benchmark::Fixture {
   storage::RecordBufferSegmentPool buffer_pool_{buffersegment_size_limit_, buffersegment_reuse_limit_};
   std::default_random_engine generator_;
   storage::LogManager *log_manager_ = DISABLED;  // logging enabled will override this value
+  common::ConcurrentBlockingQueue<storage::BufferedLogWriter *> empty_buffer_queue_;
 
   // Settings for log manager
   const uint64_t num_log_buffers_ = 100;
@@ -173,10 +174,10 @@ BENCHMARK_DEFINE_F(TPCCBenchmark, ScaleFactor4WithLogging)(benchmark::State &sta
     unlink(noisepage::BenchmarkConfig::logfile_path.data());
     thread_registry_ = new common::DedicatedThreadRegistry(DISABLED);
     // we need transactions, TPCC database, and GC
-    log_manager_ =
-        new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
-                                log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
-                                common::ManagedPointer(&buffer_pool_), common::ManagedPointer(thread_registry_));
+    log_manager_ = new storage::LogManager(
+        noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_, log_serialization_interval_,
+        log_persist_interval_, log_persist_threshold_, common::ManagedPointer(&buffer_pool_),
+        common::ManagedPointer(&empty_buffer_queue_), DISABLED, common::ManagedPointer(thread_registry_));
     log_manager_->Start();
     transaction::TimestampManager timestamp_manager;
     transaction::DeferredActionManager deferred_action_manager{common::ManagedPointer(&timestamp_manager)};
@@ -275,10 +276,10 @@ BENCHMARK_DEFINE_F(TPCCBenchmark, ScaleFactor4WithLoggingAndMetrics)(benchmark::
     metrics_manager->EnableMetric(metrics::MetricsComponent::LOGGING);
     thread_registry_ = new common::DedicatedThreadRegistry{common::ManagedPointer(metrics_manager)};
     // we need transactions, TPCC database, and GC
-    log_manager_ =
-        new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
-                                log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
-                                common::ManagedPointer(&buffer_pool_), common::ManagedPointer(thread_registry_));
+    log_manager_ = new storage::LogManager(
+        noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_, log_serialization_interval_,
+        log_persist_interval_, log_persist_threshold_, common::ManagedPointer(&buffer_pool_),
+        common::ManagedPointer(&empty_buffer_queue_), DISABLED, common::ManagedPointer(thread_registry_));
     log_manager_->Start();
     transaction::TimestampManager timestamp_manager;
     transaction::DeferredActionManager deferred_action_manager{common::ManagedPointer(&timestamp_manager)};

benchmark/metrics/logging_metrics_benchmark.cpp

@@ -29,6 +29,7 @@ class LoggingMetricsBenchmark : public benchmark::Fixture {
   const std::chrono::microseconds gc_period_{1000};
   const std::chrono::microseconds metrics_period_{10000};
   common::DedicatedThreadRegistry *thread_registry_ = nullptr;
+  common::ConcurrentBlockingQueue<storage::BufferedLogWriter *> empty_buffer_queue_;
 
   // Settings for log manager
   const uint64_t num_log_buffers_ = 100;
@@ -55,10 +56,11 @@ BENCHMARK_DEFINE_F(LoggingMetricsBenchmark, TPCCish)(benchmark::State &state) {
     metrics_manager->EnableMetric(metrics::MetricsComponent::LOGGING);
     thread_registry_ = new common::DedicatedThreadRegistry(common::ManagedPointer(metrics_manager));
 
-    log_manager_ = new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
-                                           log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
-                                           common::ManagedPointer(&buffer_pool_),
-                                           common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
+    log_manager_ =
+        new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
+                                log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
+                                common::ManagedPointer(&buffer_pool_), common::ManagedPointer(&empty_buffer_queue_),
+                                DISABLED, common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
     log_manager_->Start();
     LargeDataTableBenchmarkObject tested(attr_sizes_, initial_table_size_, txn_length, insert_update_select_ratio,
                                          &block_store_, &buffer_pool_, &generator_, true, log_manager_);
@@ -106,10 +108,11 @@ BENCHMARK_DEFINE_F(LoggingMetricsBenchmark, HighAbortRate)(benchmark::State &sta
     metrics_manager->EnableMetric(metrics::MetricsComponent::LOGGING);
     thread_registry_ = new common::DedicatedThreadRegistry(common::ManagedPointer(metrics_manager));
 
-    log_manager_ = new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
-                                           log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
-                                           common::ManagedPointer(&buffer_pool_),
-                                           common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
+    log_manager_ =
+        new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
+                                log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
+                                common::ManagedPointer(&buffer_pool_), common::ManagedPointer(&empty_buffer_queue_),
+                                DISABLED, common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
     log_manager_->Start();
     LargeDataTableBenchmarkObject tested(attr_sizes_, 1000, txn_length, insert_update_select_ratio, &block_store_,
                                          &buffer_pool_, &generator_, true, log_manager_);
@@ -157,10 +160,11 @@ BENCHMARK_DEFINE_F(LoggingMetricsBenchmark, SingleStatementInsert)(benchmark::St
     metrics_manager->EnableMetric(metrics::MetricsComponent::LOGGING);
     thread_registry_ = new common::DedicatedThreadRegistry(common::ManagedPointer(metrics_manager));
 
-    log_manager_ = new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
-                                           log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
-                                           common::ManagedPointer(&buffer_pool_),
-                                           common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
+    log_manager_ =
+        new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
+                                log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
+                                common::ManagedPointer(&buffer_pool_), common::ManagedPointer(&empty_buffer_queue_),
+                                DISABLED, common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
     log_manager_->Start();
     LargeDataTableBenchmarkObject tested(attr_sizes_, 0, txn_length, insert_update_select_ratio, &block_store_,
                                          &buffer_pool_, &generator_, true, log_manager_);
@@ -208,10 +212,11 @@ BENCHMARK_DEFINE_F(LoggingMetricsBenchmark, SingleStatementUpdate)(benchmark::St
     metrics_manager->EnableMetric(metrics::MetricsComponent::LOGGING);
     thread_registry_ = new common::DedicatedThreadRegistry(common::ManagedPointer(metrics_manager));
 
-    log_manager_ = new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
-                                           log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
-                                           common::ManagedPointer(&buffer_pool_),
-                                           common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
+    log_manager_ =
+        new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
+                                log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
+                                common::ManagedPointer(&buffer_pool_), common::ManagedPointer(&empty_buffer_queue_),
+                                DISABLED, common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
     log_manager_->Start();
     LargeDataTableBenchmarkObject tested(attr_sizes_, initial_table_size_, txn_length, insert_update_select_ratio,
                                          &block_store_, &buffer_pool_, &generator_, true, log_manager_);
@@ -259,10 +264,11 @@ BENCHMARK_DEFINE_F(LoggingMetricsBenchmark, SingleStatementSelect)(benchmark::St
     metrics_manager->EnableMetric(metrics::MetricsComponent::LOGGING);
     thread_registry_ = new common::DedicatedThreadRegistry(common::ManagedPointer(metrics_manager));
 
-    log_manager_ = new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
-                                           log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
-                                           common::ManagedPointer(&buffer_pool_),
-                                           common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
+    log_manager_ =
+        new storage::LogManager(noisepage::BenchmarkConfig::logfile_path.data(), num_log_buffers_,
+                                log_serialization_interval_, log_persist_interval_, log_persist_threshold_,
+                                common::ManagedPointer(&buffer_pool_), common::ManagedPointer(&empty_buffer_queue_),
+                                DISABLED, common::ManagedPointer<common::DedicatedThreadRegistry>(thread_registry_));
     log_manager_->Start();
     LargeDataTableBenchmarkObject tested(attr_sizes_, initial_table_size_, txn_length, insert_update_select_ratio,
                                          &block_store_, &buffer_pool_, &generator_, true, log_manager_);