Add env_fault_injection argument to db_stress (facebook#6687)

Summary: Add env_fault_injection argument to db_stress. When enabled, FaultInjectionTestEnv will be used instead. Currently this option does not support running with other env setting. This will allow us to later manually produce error when running db_crashtest. Pull Request resolved: facebook#6687 Test Plan: make db_stress -j32 ./db_stress --env_fault_injection ./db_stress --env_fault_injection --hdfs // expect error message Reviewed By: ajkr Differential Revision: D21014683 Pulled By: yhchiang fbshipit-source-id: 0724aeac37efd57adb72a37defe6dbd3bfa8106a
wxing1292 · Apr 16, 2020 · 5801af4 · 5801af4
1 parent 2767972
commit 5801af4
Show file tree

Hide file tree

Showing 7 changed files with 20 additions and 11 deletions.
diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h
@@ -70,6 +70,7 @@
 #include "util/string_util.h"
 #include "utilities/blob_db/blob_db.h"
 #include "test_util/testutil.h"
+#include "test_util/fault_injection_test_env.h"
 
 #include "utilities/merge_operators.h"
 
@@ -229,6 +230,7 @@ DECLARE_bool(blob_db_enable_gc);
 DECLARE_double(blob_db_gc_cutoff);
 #endif  // !ROCKSDB_LITE
 DECLARE_int32(approximate_size_one_in);
+DECLARE_bool(sync_fault_injection);
 
 const long KB = 1024;
 const int kRandomValueMaxFactor = 3;

diff --git a/db_stress_tool/db_stress_driver.cc b/db_stress_tool/db_stress_driver.cc
@@ -61,6 +61,12 @@ bool RunStressTest(StressTest* stress) {
     stress->InitReadonlyDb(&shared);
   }
 
+#ifndef NDEBUG
+  if (FLAGS_sync_fault_injection) {
+    fault_fs_guard->SetFilesystemDirectWritable(false);
+  }
+#endif
+
   uint32_t n = shared.GetNumThreads();
 
   uint64_t now = db_stress_env->NowMicros();

diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc
@@ -674,4 +674,8 @@ DEFINE_int32(approximate_size_one_in, 64,
 
 DEFINE_int32(read_fault_one_in, 1000,
             "On non-zero, enables fault injection on read");
+
+DEFINE_bool(sync_fault_injection, false,
+            "If true, FaultInjectionTestFS will be used for write operations, "
+            " and unsynced data in DB will lost after crash.");
 #endif  // GFLAGS
diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc
@@ -1728,6 +1728,7 @@ void StressTest::PrintEnv() const {
   fprintf(stdout, "Use dynamic level         : %d\n",
           static_cast<int>(FLAGS_level_compaction_dynamic_level_bytes));
   fprintf(stdout, "Read fault one in         : %d\n", FLAGS_read_fault_one_in);
+  fprintf(stdout, "Sync fault injection      : %d\n", FLAGS_sync_fault_injection);
 
   fprintf(stdout, "------------------------------------------------\n");
 }

diff --git a/db_stress_tool/db_stress_tool.cc b/db_stress_tool/db_stress_tool.cc
@@ -73,8 +73,9 @@ int db_stress_tool(int argc, char** argv) {
   } else {
     raw_env = Env::Default();
   }
+
 #ifndef NDEBUG
-  if (FLAGS_read_fault_one_in) {
+  if (FLAGS_read_fault_one_in || FLAGS_sync_fault_injection) {
     FaultInjectionTestFS* fs =
         new FaultInjectionTestFS(raw_env->GetFileSystem());
     fault_fs_guard.reset(fs);
@@ -84,6 +85,7 @@ int db_stress_tool(int argc, char** argv) {
     raw_env = fault_env_guard.get();
   }
 #endif
+
   env_wrapper_guard = std::make_shared<DbStressEnvWrapper>(raw_env);
   db_stress_env = env_wrapper_guard.get();
 

diff --git a/test_util/fault_injection_test_fs.cc b/test_util/fault_injection_test_fs.cc
@@ -241,15 +241,8 @@ IOStatus FaultInjectionTestFS::NewWritableFile(
   if (IsFilesystemDirectWritable()) {
     return target()->NewWritableFile(fname, file_opts, result, dbg);
   }
-  // Not allow overwriting files
-  IOStatus io_s = target()->FileExists(fname, IOOptions(), dbg);
-  if (io_s.ok()) {
-    return IOStatus::Corruption("File already exists.");
-  } else if (!io_s.IsNotFound()) {
-    assert(io_s.IsIOError());
-    return io_s;
-  }
-  io_s = target()->NewWritableFile(fname, file_opts, result, dbg);
+
+  IOStatus io_s = target()->NewWritableFile(fname, file_opts, result, dbg);
   if (io_s.ok()) {
     result->reset(new TestFSWritableFile(fname, std::move(*result), this));
     // WritableFileWriter* file is opened

diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py
@@ -114,7 +114,8 @@
     "continuous_verification_interval" : 0,
     "max_key_len": 3,
     "key_len_percent_dist": "1,30,69",
-    "read_fault_one_in": lambda: random.choice([0, 1000])
+    "read_fault_one_in": lambda: random.choice([0, 1000]),
+    "sync_fault_injection": False
 }
 
 _TEST_DIR_ENV_VAR = 'TEST_TMPDIR'