Fix the download path for ingest (vesoft-inc#927)

* fix download and ingest * fix test case
walterlife · Sep 17, 2019 · 2e6df7d · 2e6df7d
1 parent fcc85bd
commit 2e6df7d
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 27 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,10 +15,14 @@ compile_commands.json
 src/common/base/Base.h.gch
 gen-*
 
+CPackConfig.cmake
+CPackSourceConfig.cmake
 CMakeCache.txt
 Makefile
 cmake_install.cmake
 CTestTestfile.cmake
 CMakeFiles/
 Testing/
 target/
+cluster.id
+pids/
diff --git a/src/daemons/MetaDaemon.cpp b/src/daemons/MetaDaemon.cpp
@@ -133,12 +133,12 @@ bool initWebService(nebula::kvstore::KVStore* kvstore,
     nebula::WebService::registerHandler("/status", [] {
         return new nebula::meta::MetaHttpStatusHandler();
     });
-    nebula::WebService::registerHandler("/download-dispatch", [&] {
+    nebula::WebService::registerHandler("/download-dispatch", [kvstore, helper, pool] {
         auto handler = new nebula::meta::MetaHttpDownloadHandler();
         handler->init(kvstore, helper, pool);
         return handler;
     });
-    nebula::WebService::registerHandler("/ingest-dispatch", [&] {
+    nebula::WebService::registerHandler("/ingest-dispatch", [kvstore, pool] {
         auto handler = new nebula::meta::MetaHttpIngestHandler();
         handler->init(kvstore, pool);
         return handler;

diff --git a/src/kvstore/NebulaStore.cpp b/src/kvstore/NebulaStore.cpp
@@ -422,14 +422,13 @@ ResultCode NebulaStore::ingest(GraphSpaceID spaceId) {
                 return error(ret);
             }
 
-            auto path = value(ret)->getDataRoot();
-            LOG(INFO) << "Ingesting Part " << part;
+            auto path = folly::stringPrintf("%s/download/%d", value(ret)->getDataRoot(), part);
             if (!fs::FileUtils::exist(path)) {
                 LOG(ERROR) << path << " not existed";
                 return ResultCode::ERR_IO_ERROR;
             }
 
-            auto files = nebula::fs::FileUtils::listAllFilesInDir(path, true, "*.sst");
+            auto files = nebula::fs::FileUtils::listAllFilesInDir(path.c_str(), true, "*.sst");
             for (auto file : files) {
                 VLOG(3) << "Ingesting extra file: " << file;
                 extras.emplace_back(file);

diff --git a/src/storage/test/StorageHttpIngestHandlerTest.cpp b/src/storage/test/StorageHttpIngestHandlerTest.cpp
@@ -26,13 +26,32 @@ class StorageHttpIngestHandlerTestEnv : public ::testing::Environment {
         rootPath_ = std::make_unique<fs::TempDir>("/tmp/StorageHttpIngestHandler.XXXXXX");
         kv_ = TestUtils::initKV(rootPath_->path(), 1);
 
+        auto partPath = folly::stringPrintf("%s/disk1/nebula/0/download/0", rootPath_->path());
+        ASSERT_TRUE(nebula::fs::FileUtils::makeDir(partPath));
+
+        auto options = rocksdb::Options();
+        auto env = rocksdb::EnvOptions();
+        rocksdb::SstFileWriter writer{env, options};
+
+        auto sstPath = folly::stringPrintf("%s/data.sst", partPath.c_str());
+        auto status = writer.Open(sstPath);
+        ASSERT_EQ(rocksdb::Status::OK(), status);
+
+        for (auto i = 0; i < 10; i++) {
+            status = writer.Put(folly::stringPrintf("key_%d", i),
+                                folly::stringPrintf("val_%d", i));
+            ASSERT_EQ(rocksdb::Status::OK(), status);
+        }
+        status = writer.Finish();
+        ASSERT_EQ(rocksdb::Status::OK(), status);
+
         WebService::registerHandler("/ingest", [this] {
             auto handler = new storage::StorageHttpIngestHandler();
             handler->init(kv_.get());
             return handler;
         });
-        auto status = WebService::start();
-        ASSERT_TRUE(status.ok()) << status;
+        auto webStatus = WebService::start();
+        ASSERT_TRUE(webStatus.ok()) << webStatus;
     }
 
     void TearDown() override {
@@ -48,26 +67,6 @@ class StorageHttpIngestHandlerTestEnv : public ::testing::Environment {
 };
 
 TEST(StorageHttpIngestHandlerTest, StorageIngestTest) {
-    auto path = "/tmp/StorageHttpIngestData.XXXXXX";
-    std::unique_ptr<fs::TempDir> externalPath_ = std::make_unique<fs::TempDir>(path);
-    auto partPath = folly::stringPrintf("%s/1", externalPath_->path());
-    ASSERT_TRUE(nebula::fs::FileUtils::makeDir(partPath));
-
-    auto options = rocksdb::Options();
-    auto env = rocksdb::EnvOptions();
-    rocksdb::SstFileWriter writer{env, options};
-    auto sstPath = folly::stringPrintf("%s/data.sst", partPath.c_str());
-    auto status = writer.Open(sstPath);
-    ASSERT_EQ(rocksdb::Status::OK(), status);
-
-    for (auto i = 0; i < 10; i++) {
-        status = writer.Put(folly::stringPrintf("key_%d", i),
-                            folly::stringPrintf("val_%d", i));
-        ASSERT_EQ(rocksdb::Status::OK(), status);
-    }
-    status = writer.Finish();
-    ASSERT_EQ(rocksdb::Status::OK(), status);
-
     {
         auto url = "/ingest?space=0";
         auto request = folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(),