From 5801af464677de9361a4b24ed0dc6cac3d4c91cb Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang Date: Thu, 16 Apr 2020 11:10:53 -0700 Subject: [PATCH] Add env_fault_injection argument to db_stress (#6687) Summary: Add env_fault_injection argument to db_stress. When enabled, FaultInjectionTestEnv will be used instead. Currently this option does not support running with other env setting. This will allow us to later manually produce error when running db_crashtest. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6687 Test Plan: make db_stress -j32 ./db_stress --env_fault_injection ./db_stress --env_fault_injection --hdfs // expect error message Reviewed By: ajkr Differential Revision: D21014683 Pulled By: yhchiang fbshipit-source-id: 0724aeac37efd57adb72a37defe6dbd3bfa8106a --- db_stress_tool/db_stress_common.h | 2 ++ db_stress_tool/db_stress_driver.cc | 6 ++++++ db_stress_tool/db_stress_gflags.cc | 4 ++++ db_stress_tool/db_stress_test_base.cc | 1 + db_stress_tool/db_stress_tool.cc | 4 +++- test_util/fault_injection_test_fs.cc | 11 ++--------- tools/db_crashtest.py | 3 ++- 7 files changed, 20 insertions(+), 11 deletions(-) diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index 7f23062c72e..a3be90fdf1c 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -70,6 +70,7 @@ #include "util/string_util.h" #include "utilities/blob_db/blob_db.h" #include "test_util/testutil.h" +#include "test_util/fault_injection_test_env.h" #include "utilities/merge_operators.h" @@ -229,6 +230,7 @@ DECLARE_bool(blob_db_enable_gc); DECLARE_double(blob_db_gc_cutoff); #endif // !ROCKSDB_LITE DECLARE_int32(approximate_size_one_in); +DECLARE_bool(sync_fault_injection); const long KB = 1024; const int kRandomValueMaxFactor = 3; diff --git a/db_stress_tool/db_stress_driver.cc b/db_stress_tool/db_stress_driver.cc index 59e019c607e..7ab202212ba 100644 --- a/db_stress_tool/db_stress_driver.cc +++ b/db_stress_tool/db_stress_driver.cc @@ -61,6 +61,12 @@ bool RunStressTest(StressTest* stress) { stress->InitReadonlyDb(&shared); } +#ifndef NDEBUG + if (FLAGS_sync_fault_injection) { + fault_fs_guard->SetFilesystemDirectWritable(false); + } +#endif + uint32_t n = shared.GetNumThreads(); uint64_t now = db_stress_env->NowMicros(); diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index f1c10b0a1db..ece78cf332b 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -674,4 +674,8 @@ DEFINE_int32(approximate_size_one_in, 64, DEFINE_int32(read_fault_one_in, 1000, "On non-zero, enables fault injection on read"); + +DEFINE_bool(sync_fault_injection, false, + "If true, FaultInjectionTestFS will be used for write operations, " + " and unsynced data in DB will lost after crash."); #endif // GFLAGS diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index 68e3102a742..6e5013faa27 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -1728,6 +1728,7 @@ void StressTest::PrintEnv() const { fprintf(stdout, "Use dynamic level : %d\n", static_cast(FLAGS_level_compaction_dynamic_level_bytes)); fprintf(stdout, "Read fault one in : %d\n", FLAGS_read_fault_one_in); + fprintf(stdout, "Sync fault injection : %d\n", FLAGS_sync_fault_injection); fprintf(stdout, "------------------------------------------------\n"); } diff --git a/db_stress_tool/db_stress_tool.cc b/db_stress_tool/db_stress_tool.cc index 9f9080336a5..925ccfcefce 100644 --- a/db_stress_tool/db_stress_tool.cc +++ b/db_stress_tool/db_stress_tool.cc @@ -73,8 +73,9 @@ int db_stress_tool(int argc, char** argv) { } else { raw_env = Env::Default(); } + #ifndef NDEBUG - if (FLAGS_read_fault_one_in) { + if (FLAGS_read_fault_one_in || FLAGS_sync_fault_injection) { FaultInjectionTestFS* fs = new FaultInjectionTestFS(raw_env->GetFileSystem()); fault_fs_guard.reset(fs); @@ -84,6 +85,7 @@ int db_stress_tool(int argc, char** argv) { raw_env = fault_env_guard.get(); } #endif + env_wrapper_guard = std::make_shared(raw_env); db_stress_env = env_wrapper_guard.get(); diff --git a/test_util/fault_injection_test_fs.cc b/test_util/fault_injection_test_fs.cc index 2658b8e3742..6e4bd84ae30 100644 --- a/test_util/fault_injection_test_fs.cc +++ b/test_util/fault_injection_test_fs.cc @@ -241,15 +241,8 @@ IOStatus FaultInjectionTestFS::NewWritableFile( if (IsFilesystemDirectWritable()) { return target()->NewWritableFile(fname, file_opts, result, dbg); } - // Not allow overwriting files - IOStatus io_s = target()->FileExists(fname, IOOptions(), dbg); - if (io_s.ok()) { - return IOStatus::Corruption("File already exists."); - } else if (!io_s.IsNotFound()) { - assert(io_s.IsIOError()); - return io_s; - } - io_s = target()->NewWritableFile(fname, file_opts, result, dbg); + + IOStatus io_s = target()->NewWritableFile(fname, file_opts, result, dbg); if (io_s.ok()) { result->reset(new TestFSWritableFile(fname, std::move(*result), this)); // WritableFileWriter* file is opened diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index e947bf604d3..ced21162f8b 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -114,7 +114,8 @@ "continuous_verification_interval" : 0, "max_key_len": 3, "key_len_percent_dist": "1,30,69", - "read_fault_one_in": lambda: random.choice([0, 1000]) + "read_fault_one_in": lambda: random.choice([0, 1000]), + "sync_fault_injection": False } _TEST_DIR_ENV_VAR = 'TEST_TMPDIR'