RocksDB Options file format and its serialization / deserialization.

Summary: This patch defines the format of RocksDB options file, which follows the INI file format, and implements functions for its serialization and deserialization. An example RocksDB options file can be found in examples/rocksdb_option_file_example.ini. A typical RocksDB options file has three sections, which are Version, DBOptions, and more than one CFOptions. The RocksDB options file in general follows the basic INI file format with the following extensions / modifications: * Escaped characters We escaped the following characters: - \n -- line feed - new line - \r -- carriage return - \\ -- backslash \ - \: -- colon symbol : - \# -- hash tag # * Comments We support # style comments. Comments can appear at the ending part of a line. * Statements A statement is of the form option_name = value. Each statement contains a '=', where extra white-spaces are supported. However, we don't support multi-lined statement. Furthermore, each line can only contain at most one statement. * Section Sections are of the form [SecitonTitle "SectionArgument"], where section argument is optional. * List We use colon-separated string to represent a list. For instance, n1:n2:n3:n4 is a list containing four values. Below is an example of a RocksDB options file: [Version] rocksdb_version=4.0.0 options_file_version=1.0 [DBOptions] max_open_files=12345 max_background_flushes=301 [CFOptions "default"] [CFOptions "the second column family"] [CFOptions "the third column family"] Test Plan: Added many tests in options_test.cc Reviewers: igor, IslamAbdelRahman, sdong, anthony Reviewed By: anthony Subscribers: maykov, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D46059
facebook · Sep 29, 2015 · 74b100a · 74b100a
1 parent 75134f7
commit 74b100a
Show file tree

Hide file tree

Showing 9 changed files with 1,506 additions and 263 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -199,6 +199,7 @@ set(SOURCES
         util/options.cc
         util/options_builder.cc
         util/options_helper.cc
+        util/options_parser.cc
         util/perf_context.cc
         util/perf_level.cc
         util/rate_limiter.cc

diff --git a/examples/rocksdb_option_file_example.ini b/examples/rocksdb_option_file_example.ini
@@ -0,0 +1,53 @@
+# This is a RocksDB option file.
+#
+# A typical RocksDB options file has three sections, which are
+# Version, DBOptions, and more than one CFOptions.  The RocksDB
+# options file in general follows the basic INI file format
+# with the following extensions / modifications:
+#
+#  * Escaped characters
+#    We escaped the following characters:
+#     - \n -- line feed - new line
+#     - \r -- carriage return
+#     - \\ -- backslash \
+#     - \: -- colon symbol :
+#     - \# -- hash tag #
+#  * Comments
+#    We support # style comments.  Comments can appear at the ending
+#    part of a line.
+#  * Statements
+#    A statement is of the form option_name = value.
+#    Each statement contains a '=', where extra white-spaces
+#    are supported. However, we don't support multi-lined statement.
+#    Furthermore, each line can only contain at most one statement.
+#  * Section
+#    Sections are of the form [SecitonTitle "SectionArgument"],
+#    where section argument is optional.
+#  * List
+#    We use colon-separated string to represent a list.
+#    For instance, n1:n2:n3:n4 is a list containing four values.
+#
+# Below is an example of a RocksDB options file:
+[Version]
+  # The Version section stores the version information about rocksdb
+  # and option file.  This is used for handling potential format
+  # change in the future.
+  rocksdb_version=4.0.0  # We support "#" style comment.
+  options_file_version=1.0
+[DBOptions]
+  # Followed by the Version section is the DBOptions section.
+  # The value of an options can be assigned using a statement.
+  # Note that for those options that is not set in the options file,
+  # we will use the default value.
+  max_open_files=12345
+  max_background_flushes=301
+[CFOptions "default"]
+  # ColumnFamilyOptions section must follow the format of
+  # [CFOptions "cf name"].  If a rocksdb instance
+  # has multiple column families, then its CFOptions must be
+  # specified in the same order as column family creation order.
+[CFOptions "the second column family"]
+  # Each column family must have one section in the RocksDB option
+  # file even all the options of this column family are set to
+  # default value.
+[CFOptions "the third column family"]
diff --git a/include/rocksdb/convenience.h b/include/rocksdb/convenience.h
@@ -14,16 +14,28 @@ namespace rocksdb {
 
 #ifndef ROCKSDB_LITE
 // Take a map of option name and option value, apply them into the
-// base_options, and return the new options as a result
+// base_options, and return the new options as a result.
+//
+// If input_strings_escaped is set to true, then each escaped characters
+// prefixed by '\' in the the values of the opts_map will be further
+// converted back to the raw string before assigning to the associated
+// options.
 Status GetColumnFamilyOptionsFromMap(
     const ColumnFamilyOptions& base_options,
     const std::unordered_map<std::string, std::string>& opts_map,
-    ColumnFamilyOptions* new_options);
+    ColumnFamilyOptions* new_options, bool input_strings_escaped = false);
 
+// Take a map of option name and option value, apply them into the
+// base_options, and return the new options as a result.
+//
+// If input_strings_escaped is set to true, then each escaped characters
+// prefixed by '\' in the the values of the opts_map will be further
+// converted back to the raw string before assigning to the associated
+// options.
 Status GetDBOptionsFromMap(
     const DBOptions& base_options,
     const std::unordered_map<std::string, std::string>& opts_map,
-    DBOptions* new_options);
+    DBOptions* new_options, bool input_strings_escaped = false);
 
 Status GetBlockBasedTableOptionsFromMap(
     const BlockBasedTableOptions& table_options,
@@ -48,11 +60,13 @@ Status GetDBOptionsFromString(
     const std::string& opts_str,
     DBOptions* new_options);
 
-Status GetStringFromDBOptions(const DBOptions& db_options,
-                              std::string* opts_str);
+Status GetStringFromDBOptions(std::string* opts_str,
+                              const DBOptions& db_options,
+                              const std::string& delimiter = ";  ");
 
-Status GetStringFromColumnFamilyOptions(const ColumnFamilyOptions& db_options,
-                                        std::string* opts_str);
+Status GetStringFromColumnFamilyOptions(std::string* opts_str,
+                                        const ColumnFamilyOptions& db_options,
+                                        const std::string& delimiter = ";  ");
 
 Status GetBlockBasedTableOptionsFromString(
     const BlockBasedTableOptions& table_options,

diff --git a/src.mk b/src.mk
@@ -140,8 +140,9 @@ LIB_SOURCES =                                                   \
   util/options_builder.cc                                       \
   util/options.cc                                               \
   util/options_helper.cc                                        \
+  util/options_parser.cc                                        \
   util/perf_context.cc                                          \
-  util/perf_level.cc                                          \
+  util/perf_level.cc                                            \
   util/rate_limiter.cc                                          \
   util/skiplistrep.cc                                           \
   util/slice.cc                                                 \

diff --git a/util/options_helper.cc b/util/options_helper.cc
@@ -22,6 +22,67 @@
 namespace rocksdb {
 
 #ifndef ROCKSDB_LITE
+bool isSpecialChar(const char c) {
+  if (c == '\\' || c == '#' || c == ':' || c == '\r' || c == '\n') {
+    return true;
+  }
+  return false;
+}
+
+char UnescapeChar(const char c) {
+  static const std::unordered_map<char, char> convert_map = {{'r', '\r'},
+                                                             {'n', '\n'}};
+
+  auto iter = convert_map.find(c);
+  if (iter == convert_map.end()) {
+    return c;
+  }
+  return iter->second;
+}
+
+char EscapeChar(const char c) {
+  static const std::unordered_map<char, char> convert_map = {{'\n', 'n'},
+                                                             {'\r', 'r'}};
+
+  auto iter = convert_map.find(c);
+  if (iter == convert_map.end()) {
+    return c;
+  }
+  return iter->second;
+}
+
+std::string EscapeOptionString(const std::string& raw_string) {
+  std::string output;
+  for (auto c : raw_string) {
+    if (isSpecialChar(c)) {
+      output += '\\';
+      output += EscapeChar(c);
+    } else {
+      output += c;
+    }
+  }
+
+  return output;
+}
+
+std::string UnescapeOptionString(const std::string& escaped_string) {
+  bool escaped = false;
+  std::string output;
+
+  for (auto c : escaped_string) {
+    if (escaped) {
+      output += UnescapeChar(c);
+      escaped = false;
+    } else {
+      if (c == '\\') {
+        escaped = true;
+        continue;
+      }
+      output += c;
+    }
+  }
+  return output;
+}
 
 namespace {
 CompressionType ParseCompressionType(const std::string& type) {
@@ -232,7 +293,8 @@ bool SerializeSingleOptionHelper(const char* opt_address,
       *value = ToString(*(reinterpret_cast<const double*>(opt_address)));
       break;
     case OptionType::kString:
-      *value = *(reinterpret_cast<const std::string*>(opt_address));
+      *value = EscapeOptionString(
+          *(reinterpret_cast<const std::string*>(opt_address)));
       break;
     case OptionType::kCompactionStyle:
       *value = CompactionStyleToString(
@@ -461,8 +523,12 @@ Status StringToMap(const std::string& opts_str,
   return Status::OK();
 }
 
-bool ParseColumnFamilyOption(const std::string& name, const std::string& value,
-                             ColumnFamilyOptions* new_options) {
+bool ParseColumnFamilyOption(const std::string& name,
+                             const std::string& org_value,
+                             ColumnFamilyOptions* new_options,
+                             bool input_string_escaped = false) {
+  const std::string& value =
+      input_string_escaped ? UnescapeOptionString(org_value) : org_value;
   try {
     if (name == "max_bytes_for_level_multiplier_additional") {
       new_options->max_bytes_for_level_multiplier_additional.clear();
@@ -573,8 +639,10 @@ bool ParseColumnFamilyOption(const std::string& name, const std::string& value,
   return true;
 }
 
-bool SerializeSingleDBOption(const DBOptions& db_options,
-                             const std::string& name, std::string* opt_string) {
+bool SerializeSingleDBOption(std::string* opt_string,
+                             const DBOptions& db_options,
+                             const std::string& name,
+                             const std::string& delimiter) {
   auto iter = db_options_type_info.find(name);
   if (iter == db_options_type_info.end()) {
     return false;
@@ -585,20 +653,21 @@ bool SerializeSingleDBOption(const DBOptions& db_options,
   std::string value;
   bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value);
   if (result) {
-    *opt_string = name + " = " + value + ";  ";
+    *opt_string = name + "=" + value + delimiter;
   }
   return result;
 }
 
-Status GetStringFromDBOptions(const DBOptions& db_options,
-                              std::string* opt_string) {
+Status GetStringFromDBOptions(std::string* opt_string,
+                              const DBOptions& db_options,
+                              const std::string& delimiter) {
   assert(opt_string);
   opt_string->clear();
   for (auto iter = db_options_type_info.begin();
        iter != db_options_type_info.end(); ++iter) {
     std::string single_output;
-    bool result =
-        SerializeSingleDBOption(db_options, iter->first, &single_output);
+    bool result = SerializeSingleDBOption(&single_output, db_options,
+                                          iter->first, delimiter);
     assert(result);
     if (result) {
       opt_string->append(single_output);
@@ -607,9 +676,10 @@ Status GetStringFromDBOptions(const DBOptions& db_options,
   return Status::OK();
 }
 
-bool SerializeSingleColumnFamilyOption(const ColumnFamilyOptions& cf_options,
+bool SerializeSingleColumnFamilyOption(std::string* opt_string,
+                                       const ColumnFamilyOptions& cf_options,
                                        const std::string& name,
-                                       std::string* opt_string) {
+                                       const std::string& delimiter) {
   auto iter = cf_options_type_info.find(name);
   if (iter == cf_options_type_info.end()) {
     return false;
@@ -620,32 +690,36 @@ bool SerializeSingleColumnFamilyOption(const ColumnFamilyOptions& cf_options,
   std::string value;
   bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value);
   if (result) {
-    *opt_string = name + " = " + value + ";  ";
+    *opt_string = name + "=" + value + delimiter;
   }
   return result;
 }
 
-Status GetStringFromColumnFamilyOptions(const ColumnFamilyOptions& cf_options,
-                                        std::string* opt_string) {
+Status GetStringFromColumnFamilyOptions(std::string* opt_string,
+                                        const ColumnFamilyOptions& cf_options,
+                                        const std::string& delimiter) {
   assert(opt_string);
   opt_string->clear();
   for (auto iter = cf_options_type_info.begin();
        iter != cf_options_type_info.end(); ++iter) {
     std::string single_output;
-    bool result = SerializeSingleColumnFamilyOption(cf_options, iter->first,
-                                                    &single_output);
+    bool result = SerializeSingleColumnFamilyOption(&single_output, cf_options,
+                                                    iter->first, delimiter);
     if (result) {
       opt_string->append(single_output);
     } else {
-      printf("failed to serialize %s\n", iter->first.c_str());
+      return Status::InvalidArgument("failed to serialize %s\n",
+                                     iter->first.c_str());
     }
     assert(result);
   }
   return Status::OK();
 }
 
-bool ParseDBOption(const std::string& name, const std::string& value,
-                   DBOptions* new_options) {
+bool ParseDBOption(const std::string& name, const std::string& org_value,
+                   DBOptions* new_options, bool input_string_escaped = false) {
+  const std::string& value =
+      input_string_escaped ? UnescapeOptionString(org_value) : org_value;
   try {
     if (name == "rate_limiter_bytes_per_sec") {
       new_options->rate_limiter.reset(
@@ -791,11 +865,12 @@ Status GetPlainTableOptionsFromMap(
 Status GetColumnFamilyOptionsFromMap(
     const ColumnFamilyOptions& base_options,
     const std::unordered_map<std::string, std::string>& opts_map,
-    ColumnFamilyOptions* new_options) {
+    ColumnFamilyOptions* new_options, bool input_strings_escaped) {
   assert(new_options);
   *new_options = base_options;
   for (const auto& o : opts_map) {
-    if (!ParseColumnFamilyOption(o.first, o.second, new_options)) {
+    if (!ParseColumnFamilyOption(o.first, o.second, new_options,
+                                 input_strings_escaped)) {
       return Status::InvalidArgument("Can't parse option " + o.first);
     }
   }
@@ -817,11 +892,11 @@ Status GetColumnFamilyOptionsFromString(
 Status GetDBOptionsFromMap(
     const DBOptions& base_options,
     const std::unordered_map<std::string, std::string>& opts_map,
-    DBOptions* new_options) {
+    DBOptions* new_options, bool input_strings_escaped) {
   assert(new_options);
   *new_options = base_options;
   for (const auto& o : opts_map) {
-    if (!ParseDBOption(o.first, o.second, new_options)) {
+    if (!ParseDBOption(o.first, o.second, new_options, input_strings_escaped)) {
       return Status::InvalidArgument("Can't parse option " + o.first);
     }
   }
@@ -860,5 +935,5 @@ Status GetOptionsFromString(const Options& base_options,
   return Status::OK();
 }
 
-#endif  // ROCKSDB_LITE
+#endif  // !ROCKSDB_LITE
 }  // namespace rocksdb