GraphScope 2/3: (graph-learn on vineyard) use vineyard fragment as the underlying graph.

.gitignore

@@ -11,7 +11,8 @@
 **/__pycache__/
 
 # build dir
-**/dist/
+graph-learn/dist/
+dynamic_graph_service/dist/
 **/build/
 **/built/
 **/cmake-build/

graphlearn/.gitignore

@@ -1,3 +1,10 @@
 graph_learn.egg-info/
 dist/
-examples/tf/*.tar.gz
+examples/tf/*.tar.gz
+
+# clangd
+.cache/
+
+# core dump
+/core
+

graphlearn/CMakeLists.txt

@@ -37,6 +37,14 @@ option (GPU
   "Enable gpu"
   OFF)
 
+option (WITH_VINEYARD
+  "Enable vineyard"
+  OFF)
+
+option (VINEYARD_USE_OID
+  "Use OID when work with vineyard graphs"
+  ON)
+
 set (GL_CXX_DIALECT
   "c++11"
   CACHE
@@ -247,6 +255,17 @@ else ()
   set (GL_KNN_FILES)
 endif ()
 
+## contrib vineyard
+if (WITH_VINEYARD)
+  message(INFO "Graph-learn is built with vineyard support.")
+  find_package (vineyard)
+  include_directories (SYSTEM ${VINEYARD_INCLUDE_DIRS})
+  add_definitions (-DWITH_VINEYARD)
+  if (VINEYARD_USE_OID)
+    add_definitions (-DVINEYARD_USE_OID)
+  endif ()
+endif ()
+
 ## cxx flags
 if (DEBUG)
   set (GL_MODE_FLAGS -DDEBUG -g)
@@ -285,12 +304,16 @@ set (GL_CXX_FLAGS
   -fPIC
   -fvisibility-inlines-hidden
   -pthread
-  -mavx
-  -msse4.2
-  -msse4.1
   # fixme: check format security
   -Wno-format-security)
 
+if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")
+  set (GL_CXX_FLAGS ${GL_CXX_FLAGS}
+    -mavx
+    -msse4.2
+    -msse4.1)
+endif ()
+
 if (APPLE)
   set (GL_CXX_FLAGS ${GL_CXX_FLAGS} -Wno-deprecated-declarations)
 endif ()
@@ -338,6 +361,11 @@ if (KNN)
     PUBLIC -lm -lquadmath -lgfortran)
 endif ()
 
+if (WITH_VINEYARD)
+  target_include_directories (graphlearn_shared PUBLIC ${VINEYARD_INCLUDE_DIRS})
+  target_link_libraries (graphlearn_shared PUBLIC ${VINEYARD_LIBRARIES})
+endif ()
+
 if (GPU)
   set (CUDA_PATH /usr/local/cuda-10.0)
 
@@ -404,6 +432,10 @@ if (TESTING)
       ${GL_KNN_DIR}/*.cpp)
     add_gl_tests (${KNN_TEST_FILES})
   endif ()
+
+  if (WITH_VINEYARD)
+    target_compile_options(vineyard_storage_unittest PRIVATE "-std=c++14")
+  endif ()
 endif()
 
 # python

graphlearn/python/c/py_client.cc

@@ -118,7 +118,11 @@ void init_client_module(py::module& m) {
           return self.Sampling(req, res);
         },
         py::arg("request"),
-        py::arg("response"));
+        py::arg("response"))
+    .def("get_own_servers",
+         [](Client& self) {
+           return py::cast(self.GetOwnServers());
+         });
 
   m.def("del_op_request", [](OpRequest* req) { delete req; });
   m.def("del_op_response", [](OpResponse* res) { delete res; });

graphlearn/python/c/py_export.cc

@@ -120,6 +120,9 @@ PYBIND11_MODULE(pywrap_graphlearn, m) {
   // getters
   m.def("get_tracker_mode", &GetGlobalFlagTrackerMode);
 
+  m.def("set_vineyard_graph_id", &SetGlobalFlagVineyardGraphID);
+  m.def("set_vineyard_ipc_socket", &SetGlobalFlagVineyardIPCSocket);
+
   py::enum_<error::Code>(m, "ErrorCode")
     .value("OK", error::Code::OK)
     .value("CANCELLED", error::Code::CANCELLED)
@@ -201,7 +204,9 @@ PYBIND11_MODULE(pywrap_graphlearn, m) {
     .def_readwrite("id_type", &io::NodeSource::id_type)
     .def_readwrite("format", &io::NodeSource::format)
     .def_readwrite("attr_info", &io::NodeSource::attr_info)
-    .def_readwrite("option", &io::NodeSource::option);
+    .def_readwrite("option", &io::NodeSource::option)
+    .def_readwrite("view_type", &io::NodeSource::view_type)
+    .def_readwrite("use_attrs", &io::NodeSource::use_attrs);
 
   py::class_<io::EdgeSource>(m, "EdgeSource")
     .def(py::init<>())
@@ -212,7 +217,9 @@ PYBIND11_MODULE(pywrap_graphlearn, m) {
     .def_readwrite("format", &io::EdgeSource::format)
     .def_readwrite("direction", &io::EdgeSource::direction)
     .def_readwrite("attr_info", &io::EdgeSource::attr_info)
-    .def_readwrite("option", &io::EdgeSource::option);
+    .def_readwrite("option", &io::EdgeSource::option)
+    .def_readwrite("view_type", &io::EdgeSource::view_type)
+    .def_readwrite("use_attrs", &io::EdgeSource::use_attrs);
 
   py::class_<Status>(m, "Status")
     .def("ok", &Status::ok)
@@ -243,7 +250,8 @@ PYBIND11_MODULE(pywrap_graphlearn, m) {
         &NewRpcClient,
         py::return_value_policy::take_ownership,
         py::arg("server_id") = -1,
-        py::arg("server_own") = false);
+        py::arg("server_own") = false,
+        py::arg("client_own") = true);
 
   init_client_module(m);
 

graphlearn/python/config.py

@@ -106,4 +106,10 @@ def set_neg_sampler_retry_times(times):
   pywrap.set_neg_sampler_retry_times(times)
 
 def set_field_delimiter(delimiter="\t"):
-  pywrap.set_field_delimiter(delimiter)
+  pywrap.set_field_delimiter(delimiter)
+
+def set_vineyard_graph_id(graph_id):
+  pywrap.set_vineyard_graph_id(graph_id)
+
+def set_vineyard_ipc_socket(ipc_socket):
+  pywrap.set_vineyard_ipc_socket(ipc_socket)

graphlearn/src/common/base/config.cc

@@ -107,6 +107,8 @@ DEFINE_INT32_GLOBAL_FLAG(NegativeSamplingRetryTimes, 5)
 DEFINE_INT32_GLOBAL_FLAG(IgnoreInvalid, 1) // 1 is True, 0 is False.
 DEFINE_INT32_GLOBAL_FLAG(LocalShardCount, 8)
 DEFINE_STRING_GLOBAL_FLAG(FieldDelimiter, "\t")
+DEFINE_INT64_GLOBAL_FLAG(VineyardGraphID, 0)
+DEFINE_STRING_GLOBAL_FLAG(VineyardIPCSocket, "/var/run/vineyard.sock")
 
 // Define the setters
 DEFINE_SET_INT32_GLOBAL_FLAG(DeployMode)
@@ -142,6 +144,8 @@ DEFINE_SET_INT32_GLOBAL_FLAG(NegativeSamplingRetryTimes)
 DEFINE_SET_INT32_GLOBAL_FLAG(IgnoreInvalid)
 DEFINE_SET_INT32_GLOBAL_FLAG(LocalShardCount)
 DEFINE_SET_STRING_GLOBAL_FLAG(FieldDelimiter)
+DEFINE_SET_INT64_GLOBAL_FLAG(VineyardGraphID)
+DEFINE_SET_STRING_GLOBAL_FLAG(VineyardIPCSocket)
 
 // Define the getters
 /// Only export flags that are needed by system.

graphlearn/src/common/base/progress.h

@@ -55,7 +55,7 @@ struct Progress {
     if (count > stage * grading) {
       char buffer[100];
       snprintf(buffer, sizeof(buffer),
-               "Progress of %s: %ld", key.c_str(), stage * grading);
+               "Progress of %s: %lld", key.c_str(), stage * grading);
       USER_LOG(buffer);
       ++stage;
     }

graphlearn/src/common/threading/runner/dynamic_worker_threadpool.h

@@ -49,9 +49,9 @@ class DynamicWorkerThreadPool : public ThreadPoolBase {
   int AddTask(Callback* callback) override;
   void WaitForIdle() override;
 
-  int GetQueueLength();
-  int GetThreadNum() const;
-  int GetBusyThreadNum() const;
+  int GetQueueLength() override;
+  int GetThreadNum() const override;
+  int GetBusyThreadNum() const override;
 
 protected:
   void SetIdleThresholdInMs(int ms);

graphlearn/src/contrib/knn/builder.cc

@@ -45,7 +45,7 @@ bool BuildKnnIndex(io::NodeStorage* storage, const IndexOption& option) {
   }
 
   index->Train(n, vectors.data());
-  index->Add(n, vectors.data(), storage->GetIds()->data());
+  index->Add(n, vectors.data(), storage->GetIds().data());
 
   op::KnnIndexManager::Instance()->Add(storage->GetSideInfo()->type, index);
   return true;

graphlearn/src/core/graph/graph.h

@@ -46,8 +46,12 @@ class Graph {
 #undef DECLARE_METHOD
 };
 
-Graph* CreateLocalGraph();
-Graph* CreateRemoteGraph();
+Graph* CreateLocalGraph(const std::string& type,
+                        const std::string& view_type,
+                        const std::string& use_attrs);
+Graph* CreateRemoteGraph(const std::string& type,
+                         const std::string& view_type,
+                         const std::string& use_attrs);
 
 }  // namespace graphlearn
 

graphlearn/src/core/graph/graph_store.cc

@@ -21,6 +21,7 @@ limitations under the License.
 #include "common/base/macros.h"
 #include "common/base/progress.h"
 #include "common/threading/sync/cond.h"
+#include "core/graph/storage/storage_mode.h"
 #include "core/io/element_value.h"
 #include "core/io/edge_loader.h"
 #include "core/io/node_loader.h"
@@ -186,7 +187,12 @@ Status GraphStore::Init(
     const std::vector<io::NodeSource>& nodes) {
 
   for (const auto& e : edges) {
-    graphs_->LookupOrCreate(e.edge_type);
+    std::string decorated_edge_view = e.src_id_type + "|" + e.dst_id_type;
+    if (!e.view_type.empty()) {
+      decorated_edge_view += "|" + e.view_type;
+    }
+    graphs_->LookupOrCreate(e.edge_type, decorated_edge_view, e.use_attrs);
+
     auto it = e_types_.find(e.edge_type);
     if (it == e_types_.end()) {
       e_types_.insert({e.edge_type, 1});
@@ -195,7 +201,7 @@ Status GraphStore::Init(
     }
   }
   for (const auto& n : nodes) {
-    noders_->LookupOrCreate(n.id_type);
+    noders_->LookupOrCreate(n.id_type, n.view_type, n.use_attrs);
     n_types_.insert({n.id_type, 1});
   }
   return Status::OK();
@@ -205,6 +211,11 @@ Status GraphStore::Load(
     const std::vector<io::EdgeSource>& edges,
     const std::vector<io::NodeSource>& nodes) {
   Init(edges, nodes);
+
+  if (io::IsVineyardStorageEnabled()) {
+    return Status::OK();
+  }
+
   Initializer<io::EdgeSource,
               io::EdgeLoader,
               io::EdgeValue,
@@ -301,7 +312,7 @@ void GraphStore::BuildLocalCount() {
   for (auto it = n_types_.begin(); it != n_types_.end(); ++it) {
     Noder* noder = noders_->LookupOrCreate(it->first);
     ::graphlearn::io::NodeStorage* storage = noder->GetLocalStorage();
-    local_count_.push_back(storage->GetIds()->size() * it->second);
+    local_count_.push_back(storage->GetIds().Size() * it->second);
   }
 }
 

graphlearn/src/core/graph/heter_dispatcher.h

@@ -26,7 +26,9 @@ namespace graphlearn {
 template <class T>
 class HeterDispatcher {
 public:
-  typedef T* (*TypeCreator)();
+  typedef T* (*TypeCreator)(const std::string& type,
+                            const std::string& view_type,
+                            const std::string& use_attrs);
 
 public:
   explicit HeterDispatcher(TypeCreator creator)
@@ -39,14 +41,16 @@ class HeterDispatcher {
     }
   }
 
-  T* LookupOrCreate(const std::string& type) {
+  T* LookupOrCreate(const std::string& type,
+                    const std::string& view_type="",
+                    const std::string& use_attrs="") {
     ScopedLocker<std::mutex> _(&mtx_);
     auto it = holder_.find(type);
     if (it != holder_.end()) {
       return it->second;
     }
 
-    T* t = creator_();
+    T* t = creator_(type, view_type, use_attrs);
     holder_[type] = t;
     return t;
   }

graphlearn/src/core/graph/local_graph.cc

@@ -24,8 +24,10 @@ namespace graphlearn {
 
 class LocalGraph : public Graph {
 public:
-  LocalGraph() {
-    storage_ = CreateGraphStorage();
+  LocalGraph(const std::string& type,
+             const std::string& view_type,
+             const std::string& use_attrs) {
+    storage_ = CreateGraphStorage(type, view_type, use_attrs);
   }
 
   virtual ~LocalGraph() {
@@ -91,8 +93,10 @@ class LocalGraph : public Graph {
   io::GraphStorage* storage_;
 };
 
-Graph* CreateLocalGraph() {
-  return new LocalGraph();
+Graph* CreateLocalGraph(const std::string& type,
+                        const std::string& view_type,
+                        const std::string& use_attrs) {
+  return new LocalGraph(type, view_type, use_attrs);
 }
 
 }  // namespace graphlearn

graphlearn/src/core/graph/local_noder.cc

@@ -28,8 +28,10 @@ namespace graphlearn {
 
 class LocalNoder : public Noder {
 public:
-  LocalNoder() {
-    storage_ = CreateNodeStorage();
+  LocalNoder(const std::string& type,
+             const std::string& view_type,
+             const std::string& use_attrs) {
+    storage_ = CreateNodeStorage(type, view_type, use_attrs);
   }
 
   virtual ~LocalNoder() {
@@ -103,8 +105,10 @@ class LocalNoder : public Noder {
   io::NodeStorage* storage_;
 };
 
-Noder* CreateLocalNoder() {
-  return new LocalNoder();
+Noder* CreateLocalNoder(const std::string& type,
+                        const std::string& view_type,
+                        const std::string& use_attrs) {
+  return new LocalNoder(type, view_type, use_attrs);
 }
 
 }  // namespace graphlearn

graphlearn/src/core/graph/noder.h

@@ -46,8 +46,12 @@ class Noder {
 #undef DECLARE_METHOD
 };
 
-Noder* CreateLocalNoder();
-Noder* CreateRemoteNoder();
+Noder* CreateLocalNoder(const std::string& type,
+                        const std::string& view_type,
+                        const std::string& use_attrs);
+Noder* CreateRemoteNoder(const std::string& type,
+                         const std::string& view_type,
+                         const std::string& use_attrs);
 
 }  // namespace graphlearn