Refactors (#258)

* Refactor 1) Refactor: renamed RocksdbEngine to RocksEngine 2) Refactor: Combined RocksdbConfigFlags and RocksdbConfigOptions to RocksEngineConfig 3) Cleaned up RocksdbConfigTest and renamed it to RocksEngineConfigTest 4) Modified third-parties build.sh files to make the build support Ubuntu 5) Modified main CMakeFiles.txt to properly handle libLZMA * Rebased and addressed @dutor's comments
vesoft-inc · Apr 9, 2019 · 4131cfc · 4131cfc
1 parent 756a3a4
commit 4131cfc
Show file tree

Hide file tree

Showing 40 changed files with 298 additions and 223 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -141,6 +141,7 @@ find_package(BISON REQUIRED)
 find_package(FLEX REQUIRED)
 find_package(Readline REQUIRED)
 find_package(NCURSES REQUIRED)
+find_package(LibLZMA MODULE)
 if(NOT asan AND NOT ENABLE_NATIVE)
     find_package(PCHSupport)
     add_compile_options(-Winvalid-pch)
@@ -236,6 +237,10 @@ set(ROCKSDB_LIBRARIES ${NEBULA_HOME}/third-party/rocksdb/_install/lib/librocksdb
 
 # All compression libraries
 set(COMPRESSION_LIBRARIES bz2 snappy zstd z)
+if (LIBLZMA_FOUND)
+    include_directories(SYSTEM ${LIBLZMA_INCLUDE_DIRS})
+    list(APPEND COMPRESSION_LIBRARIES ${LIBLZMA_LIBRARIES})
+endif()
 
 if (NOT ENABLE_JEMALLOC)
     set(JEMALLOC_LIB )
@@ -262,9 +267,9 @@ macro(nebula_link_libraries target)
         ${OPENSSL_CRYPTO_LIBRARY}
         ${KRB5_LIBRARIES}
         ${COMPRESSION_LIBRARIES}
-        dl
         ${JEMALLOC_LIB}
         ${LIBUNWIND_LIBRARIES}
+        dl
         -pthread
     )
 endmacro(nebula_link_libraries)

diff --git a/src/kvstore/CMakeLists.txt b/src/kvstore/CMakeLists.txt
@@ -1,10 +1,10 @@
 add_library(
     kvstore_obj OBJECT
     Part.cpp
-    RocksdbEngine.cpp
+    RocksEngine.cpp
     PartManager.cpp
     NebulaStore.cpp
-    RocksdbConfigOptions.cpp
+    RocksEngineConfig.cpp
 )
 add_dependencies(kvstore_obj base_obj meta_client)
 

diff --git a/src/kvstore/KVEngine.h b/src/kvstore/KVEngine.h
@@ -10,7 +10,6 @@
 #include "base/Base.h"
 #include "kvstore/Common.h"
 #include "kvstore/KVIterator.h"
-#include "kvstore/RocksdbConfigOptions.h"
 
 namespace nebula {
 namespace kvstore {

diff --git a/src/kvstore/NebulaStore.cpp b/src/kvstore/NebulaStore.cpp
@@ -4,13 +4,14 @@
  *  (found in the LICENSE.Apache file in the root directory)
  */
 
+#include "base/Base.h"
 #include "kvstore/NebulaStore.h"
 #include <folly/Likely.h>
 #include <algorithm>
 #include <cstdint>
 #include "network/NetworkUtils.h"
 #include "fs/FileUtils.h"
-#include "kvstore/RocksdbEngine.h"
+#include "kvstore/RocksEngine.h"
 
 DEFINE_string(engine_type, "rocksdb", "rocksdb, memory...");
 DEFINE_string(part_type, "simple", "simple, consensus...");
@@ -71,23 +72,24 @@ KVStore* KVStore::instance(KVOptions options) {
     return instance;
 }
 
+
 Engine NebulaStore::newEngine(GraphSpaceID spaceId, std::string rootPath) {
     if (FLAGS_engine_type == "rocksdb") {
         auto dataPath = KV_DATA_PATH_FORMAT(rootPath.c_str(), spaceId);
         auto engine = std::make_pair(
                                 std::unique_ptr<KVEngine>(
-                                    new RocksdbEngine(
-                                          spaceId,
-                                          std::move(dataPath),
-                                          options_.mergeOp_,
-                                          options_.cfFactory_)),
+                                    new RocksEngine(spaceId,
+                                                    std::move(dataPath),
+                                                    options_.mergeOp_,
+                                                    options_.cfFactory_)),
                                 std::move(rootPath));
         return engine;
     } else {
         LOG(FATAL) << "Unknown Engine type " << FLAGS_engine_type;
     }
 }
 
+
 std::unique_ptr<Part> NebulaStore::newPart(GraphSpaceID spaceId,
                                            PartitionID partId,
                                            const Engine& engine) {
@@ -102,6 +104,7 @@ std::unique_ptr<Part> NebulaStore::newPart(GraphSpaceID spaceId,
     }
 }
 
+
 void NebulaStore::init() {
     CHECK(!!partMan_);
     LOG(INFO) << "Scan the local path, and init the kvs_";
@@ -157,6 +160,7 @@ void NebulaStore::init() {
     partMan_->registerHandler(this);
 }
 
+
 void NebulaStore::addSpace(GraphSpaceID spaceId) {
     folly::RWSpinLock::WriteHolder wh(&lock_);
     if (this->kvs_.find(spaceId) != this->kvs_.end()) {
@@ -171,6 +175,7 @@ void NebulaStore::addSpace(GraphSpaceID spaceId) {
     return;
 }
 
+
 void NebulaStore::addPart(GraphSpaceID spaceId, PartitionID partId) {
     folly::RWSpinLock::WriteHolder wh(&lock_);
     auto spaceIt = this->kvs_.find(spaceId);
@@ -199,6 +204,7 @@ void NebulaStore::addPart(GraphSpaceID spaceId, PartitionID partId) {
     return;
 }
 
+
 void NebulaStore::removeSpace(GraphSpaceID spaceId) {
     folly::RWSpinLock::WriteHolder wh(&lock_);
     auto spaceIt = this->kvs_.find(spaceId);
@@ -215,6 +221,7 @@ void NebulaStore::removeSpace(GraphSpaceID spaceId) {
     LOG(INFO) << "Space " << spaceId << " has been removed!";
 }
 
+
 void NebulaStore::removePart(GraphSpaceID spaceId, PartitionID partId) {
     folly::RWSpinLock::WriteHolder wh(&lock_);
     auto spaceIt = this->kvs_.find(spaceId);
@@ -229,13 +236,15 @@ void NebulaStore::removePart(GraphSpaceID spaceId, PartitionID partId) {
     LOG(INFO) << "Space " << spaceId << ", part " << partId << " has been removed!";
 }
 
+
 ResultCode NebulaStore::get(GraphSpaceID spaceId, PartitionID partId,
                             const std::string& key,
                             std::string* value) {
     CHECK_AND_RETURN_ENGINE(spaceId, partId);
     return engine->get(key, value);
 }
 
+
 ResultCode NebulaStore::range(GraphSpaceID spaceId, PartitionID partId,
                               const std::string& start,
                               const std::string& end,
@@ -244,20 +253,23 @@ ResultCode NebulaStore::range(GraphSpaceID spaceId, PartitionID partId,
     return engine->range(start, end, iter);
 }
 
+
 ResultCode NebulaStore::prefix(GraphSpaceID spaceId, PartitionID partId,
                                const std::string& prefix,
                                std::unique_ptr<KVIterator>* iter) {
     CHECK_AND_RETURN_ENGINE(spaceId, partId);
     return engine->prefix(prefix, iter);
 }
 
+
 void NebulaStore::asyncMultiPut(GraphSpaceID spaceId, PartitionID partId,
                                 std::vector<KV> keyValues,
                                 KVCallback cb) {
     CHECK_FOR_WRITE(spaceId, partId, cb);
     return partIt->second->asyncMultiPut(std::move(keyValues), std::move(cb));
 }
 
+
 void NebulaStore::asyncRemove(GraphSpaceID spaceId,
                               PartitionID partId,
                               const std::string& key,
@@ -266,6 +278,7 @@ void NebulaStore::asyncRemove(GraphSpaceID spaceId,
     return partIt->second->asyncRemove(key, std::move(cb));
 }
 
+
 void NebulaStore::asyncRemoveRange(GraphSpaceID spaceId,
                                    PartitionID partId,
                                    const std::string& start,
@@ -275,6 +288,7 @@ void NebulaStore::asyncRemoveRange(GraphSpaceID spaceId,
     return partIt->second->asyncRemoveRange(start, end, std::move(cb));
 }
 
+
 ResultCode NebulaStore::ingest(GraphSpaceID spaceId,
                                const std::string& extra,
                                const std::vector<std::string>& files) {
@@ -301,6 +315,7 @@ ResultCode NebulaStore::ingest(GraphSpaceID spaceId,
     return ResultCode::SUCCEEDED;
 }
 
+
 ResultCode NebulaStore::setOption(GraphSpaceID spaceId,
                                   const std::string& config_key,
                                   const std::string& config_value) {

diff --git a/src/kvstore/NebulaStore.h b/src/kvstore/NebulaStore.h
@@ -7,9 +7,9 @@
 #ifndef KVSTORE_NEBULASTORE_H_
 #define KVSTORE_NEBULASTORE_H_
 
+#include "base/Base.h"
 #include <gtest/gtest_prod.h>
 #include <folly/RWSpinLock.h>
-#include "base/Base.h"
 #include "kvstore/KVStore.h"
 #include "kvstore/PartManager.h"
 #include "kvstore/Part.h"
@@ -26,9 +26,10 @@ struct GraphSpaceKV {
     std::vector<Engine> engines_;
 };
 
+
 class NebulaStore : public KVStore, public Handler {
-    FRIEND_TEST(KVStoreTest, SimpleTest);
-    FRIEND_TEST(KVStoreTest, PartsTest);
+    FRIEND_TEST(NebulaStoreTest, SimpleTest);
+    FRIEND_TEST(NebulaStoreTest, PartsTest);
 
 public:
     explicit NebulaStore(KVOptions options)

diff --git a/src/kvstore/PartManager.cpp b/src/kvstore/PartManager.cpp
@@ -4,7 +4,6 @@
  *  (found in the LICENSE.Apache file in the root directory)
  */
 #include "kvstore/PartManager.h"
-#include "kvstore/RocksdbConfigFlags.h"
 
 namespace nebula {
 namespace kvstore {

diff --git a/src/kvstore/PartManager.h b/src/kvstore/PartManager.h
@@ -67,8 +67,8 @@ class PartManager {
 : * Memory based PartManager, it is used in UTs now.
  * */
 class MemPartManager final : public PartManager {
-    FRIEND_TEST(KVStoreTest, SimpleTest);
-    FRIEND_TEST(KVStoreTest, PartsTest);
+    FRIEND_TEST(NebulaStoreTest, SimpleTest);
+    FRIEND_TEST(NebulaStoreTest, PartsTest);
 
 public:
     MemPartManager() = default;