UNSW-database · Katherine2013 · Apr 24, 2019 · Apr 24, 2019 · Apr 24, 2019 · Apr 24, 2019
diff --git a/Cargo.toml b/Cargo.toml
@@ -3,11 +3,11 @@ name = "rust_graph"
 version = "0.1.9"
 authors = ["Zhengyi Yang <zhengyi.yang@outlook.com>"]
 autoexamples = true
+edition="2018"
 
 [features]
 default = []
 usize_id = []
-ldbc = ["regex"]
 
 [dependencies]
 indexmap = { version = "1.0.2",features = ["serde-1"] }
@@ -20,17 +20,39 @@ bincode = "1.0.1"
 log = "0.4"
 csv = "1"
 counter = "0.4.3"
-regex = {version = "1", optional = true }
-fnv = "1.0.6"
+serde_cbor = "0.9.0"
 fixedbitset = "0.1.9"
+hashbrown = {version = "0.2.0", features = ["serde"] }
+rayon = "1.0.3"
+serde_json = "1.0.39"
+fxhash = "0.2.1"
+rocksdb = "0.12.2"
+lru = "0.1.15"
+scoped_threadpool = "0.1.9"
+# tikv-client = { git = "https://github.com/tikv/client-rust.git" }
+tikv-client = { git = "https://github.com/tz70s/client-rust.git", branch = "fix-raw-scan-range" }
+futures-preview = { version = "0.3.0-alpha.17", features = ["compat"] }
+walkdir = "2.0.0"
+tokio = "=0.2.13"
+#protobuf = { version = "2.0"}
 
+#
+#[dependencies.hdfs]
+#git="https://github.com/UNSW-database/hdfs-rs.git"
+#default-features = false
+#optional = true
 
 [dev-dependencies]
 tempfile = "3.0.4"
-pbr = "1.0.1"
 clap = "2.32.0"
+criterion = "0.2"
+tempdir = "0.3.7"
 
-[[example]]
-name = "ldbc_to_graphmap"
-required-features = ["ldbc"]
+#[patch.crates-io]
+#hdfs = {}
 
+[build-dependencies]
+protoc-rust = "2.0"
+
+[patch.crates-io]
+raft-proto = { git = "https://github.com/tikv/raft-rs", rev = "e624c1d48460940a40d8aa69b5329460d9af87dd" }
diff --git a/LICENSE b/LICENSE
@@ -1,18 +1,18 @@
-Copyright (c) 2018 UNSW Sydney, Data and Knowledge Group.
-
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied.  See the License for the
-specific language governing permissions and limitations
-under the License.
+Copyright (c) 2018 UNSW Sydney, Data and Knowledge Group.
+ 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+ 
+  http://www.apache.org/licenses/LICENSE-2.0
+ 
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
diff --git a/README.md b/README.md
@@ -1,3 +1,118 @@
 # rust\_graph\_lib
 
-A graph libary written in Rust. 
+A graph libary written in Rust. Note that You need to install cmake, g++, clang, golang on your machine first and use Rust nightly to build this project.
+
+## Setup for hdfs reading support
+
+### 0. Explanations for `build` and `running` stage in `hdfs_lib`
+The function for reading files from `hdfs` is based on a library [`hdfs-rs`](https://github.com/hyunsik/hdfs-rs). Because the library is not update for a few years, so I fixed some bugs in the source code and stored in `src/io/hdfs//hdfs_lib`. The project is regard it as a local crate. (Just as `Cago.toml` shows: `hdfs={path="src//io//hdfs//hdfs_lib"}`).  
+* In the library, we were calling`libhdfs C APIs`[(docs here)](http://hadoop.apache.org/docs/r3.0.0/hadoop-project-dist/hadoop-hdfs/LibHdfs.html) (supported by hadoop) to implement functions. And encapsulate the `libhdfs C APIs` in the library. 
+* In the path `hdfs_lib/src/native`, there are static library(`libhdfs.a`) and shared object(`libhdfs.so`) for calling `C APIs` in rust. It helps to guarantee that the project will compile successfully even without the hadoop environment.  
+* In the file `hdfs_lib/build.rs`, we use `build.rs`[(docs here)](https://doc.rust-lang.org/cargo/reference/build-scripts.html#outputs-of-the-build-script) to pass environment variable `rustc-link-search` to prompt compiler to find the static and shared object.  
+* In the running time for calling `libhdfs C APIs`, it will using `libhdfs.so`,`libjvm.so`,`Java Environment` and `Hadoop jars`.So,
+please ensure that you have finished the following steps, if you want to use the functions for `hdfs`.
+
+### 1. Download hadoop and environment variables
+#### 1.1 Requirement:
+* Hadoop version >= [2.6.5](http://mirror.bit.edu.cn/apache/hadoop/common/hadoop-2.6.5/)
+* Java >=1.8
+* Linux Environment
+* In the running time of `libhdfs C APIs`
+* Checking in the hadoop you have installed contains `libhdfs.so` in path `$HADOOP_HOME/lib/native/`.(In the pre-build version, hadoop contains it by default)
+* Checking in the java you have installed contains `libjvm.so` in path `$JAVA_HOME/jre/lib/amd64/server/`.  
+
+#### 1.2 Environment variables:  
+Edit shell environment as following command:
+```
+vim ~/.bashrc
+
+# Change the path to your own and appending to the file
+export JAVA_HOME=/{YOUR_JAVA_INSTALLED_PATH}
+export JRE_HOME=${JAVA_HOME}/jre 
+export HADOOP_HOME=/{YOUR_HADOOP_INSTALLED_PATH} 
+export LD_LIBRARY_PATH=${HADOOP_HOME}/lib/native:${JAVA_HOME}/jre/lib/amd64/server #for libhdfs.o linking
+CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib 
+CLASSPATH=${CLASSPATH}":"`find ${HADOOP_HOME}/share/hadoop | awk '{path=path":"$0}END{print path}'` # hadoop's jars 
+export CLASSPATH 
+export PATH=${JAVA_HOME}/bin:$HOME/.cargo/bin:$HADOOP_HOME/sbin:$HADOOP_HOME/bin:$PATH
+
+#flush the environment variable to all shell session
+source ~/.bashrc
+```
+
+### 2. Configuring a pseudo hadoop and hdfs environment
+Of course, you can build a real cluster by yourself. What we need in the code is the `hdfs path` and `port`.  
+
+First of all, entering the configure files directory:`cd $HADOOP_HOME/etc/hadoop`  
+
+#### 2.1 Configure `core-site.xml`  
+```xml
+<configuration>
+<property>
+    <name>hadoop.tmp.dir</name>
+    <value>file:/usr/local/hadoop/tmp</value>
+    <description>Abase for other temporary directories.</description>
+</property>
+<property>
+    <name>fs.defaultFS</name>
+    <value>hdfs://localhost:9000</value>
+</property>
+<property>
+    <name>hadoop.http.staticuser.user</name>
+    <value>cy</value>
+</property>
+</configuration>
+```  
+
+#### 2.2 Configure `hdfs-site.xml`   
+```xml
+<configuration>
+  <property>
+    <name>dfs.replication</name>
+    <value>1</value>
+  </property>
+  <property>
+    <name>dfs.namenode.name.dir</name>
+    <value>file:/usr/local/hadoop/tmp/dfs/name</value>
+  </property>
+  <property>
+    <name>dfs.datanode.data.dir</name>
+    <value>file:/usr/local/hadoop/tmp/dfs/data</value>
+  </property>
+  <property>
+     <name>dfs.permissions.enabled</name>
+     <value>false</value>
+  </property>
+</configuration>
+```  
+
+#### 2.3 Configure `hadoop-env.sh` (Non-essential. Only for JAVA_HOME can't find during starting hdfs)
+```
+export JAVA_HOME=/{YOUR_JAVA_INSTALLED_PATH}
+```  
+### 3. Starting hdfs and checking hdfs status  
+* Starting hdfs: `./$HADOOP_HOME/sbin/start-dfs.sh`  
+You'll get a output as following if you are successful:  
+```
+Starting namenodes on [localhost]
+Starting datanodes
+Starting secondary namenodes [{MACHINE-NAME}]
+``` 
+* And you can use `jps` command to verify hdfs status.
+```
+jps
+17248 Jps
+16609 DataNode
+16482 NameNode
+4198 Main
+16847 SecondaryNameNode
+```
+* Now you can open a explorer to visit hdfs page  
+`http://localhost:9870/dfshealth.html#tab-overview`  
+The port maybe different in different version hadoop. Please check on hadoop website. 
+
+### 4. Testing and using hdfs support
+For now, you can using `hdfs support` in this library to read from local pseudo hdfs cluster(or real hdfs cluster).
+* In order to avoid tests failure in this library. We mark the tests for `hdfs support` as `ignore`.  
+So, if you want to test them, please using the following command to test the `hdfs support` independently:  
+`cargo test -- --ignored`  
diff --git a/bench-tikv-rocksdb.md b/bench-tikv-rocksdb.md
@@ -0,0 +1,92 @@
+# Benchmark TiKV and Rocksdb
+
+## 1. Benchmark TiKV and Rocksdb on a single machine
+I have deployed two pd-servers and each pd-server manages one tikv-server.
+
+(1) The following tests are all based on 100 operations and we record the average time for each operation.
+### Insert raw node/edge property operation 
+|    TiKV     |    Rocksdb    |
+|------------ |---------------|
+|    26~34ms  |    55~105ms   |
+
+### Extend one raw node/edge property operation
+|  TiKV        |    Rocksdb      |
+|--------------|-----------------|
+|  0.24~0.34ms |    0.4~1.3ms    |
+
+### Get node/edge property(all) operation
+Note that this is not a fair comparison because the Rocksdb's `get` operation in `rust_graph_lib` is simply fetch k-v from memory(it reads all k-v pairs into memory when connecting to it, it is more like the `tikv`'s `batch_get` operation) while `TiKV` needs to connect to pd-server and reads data from disk into memory to return it. With the connection time and reading all kv pairs into memory time counted in, the single get operation for rocksdb is actually around `105ms`.
+
+|  TiKV  |    Rocksdb   |
+|--------|--------------|
+|  3~4ms |  0.03~0.06ms |
+
+(2) The following `batch_get` operation test is based on batchly get 1000 keys and we record the average time.
+### Batch get node/edge property(all) operation
+ |   TiKV   | 
+ |----------|
+ |  0.008ms | 
+
+## 2. Benchmark TiKV on a cluster
+I have deployed two pd-servers and each pd-server manages four tikv-servers(totally there are two pd-servers and eight tikv-servers and they are all on different machines).
+
+### Insert raw node/edge property operation 
+|    TiKV     |   
+|------------ |
+|     90ms    | 
+
+### Extend one raw node/edge property operation
+|    TiKV    |
+|------------|
+|  0.6~0.9ms | 
+
+### Get node/edge property(all) operation
+|  TiKV  |
+|--------|
+|  4~5ms |
+
+### Batch get node/edge property(all) operation
+|       TiKV        |
+|-------------------|
+|  0.008ms ~ 0.01ms |
+
+(Batch get 1000 node/edge properties, and it takes 0.008s ~ 0.01s in total)
+
+## 3. `Batch` operations performance comparing between TiKV on a cluster and RocksDB
+I have deployed three pd-servers and each pd-server manages three tikv-servers(totally there are three pd-servers and six tikv-servers and they are all on different machines).
+And using one server to running test program.
+
+### Batch put operation on DG10
+1. current_thread::Runtime  
+
+|Batch Size|TiKV|RocksDB|
+|---|---|---|
+|100|610.306s|873.912s|
+|500|450.900s|774.325s|
+|1000|531.744s|755.248s|
+|10000|1739.592s|751.675s|
+
+2. tokio::Runtime(ThreadPool)  
+
+|Batch Size|TiKV|RocksDB|
+|---|---|---|
+|100|357.481s|776.099s|
+|500|427.291s|873.144s|
+|1000|528.148s|786.991s|
+|10000|2039.108s|784.073s|
+
+### Batch put operation on DG60
+|Batch Size|TiKV|RocksDB|
+|---|---|---|
+|100|2525.971s|13516.203s|
+|500|2666.998s|8638.527s|
+|1000|2654.622s|8247.292s|
+|10000|4054.167s|8104.632s|
+
+### Batch get node/edge property(all) operation
+|Batch Size|TiKV|RocksDB|
+|---|---|---|
+|100|0.078s ~ 0.079s|0.387s ~ 0.425s|
+|500|0.382s ~ 0.387s|1.848s ~ 1.938s|
+|1000|0.468s ~ 0.476s|3.869s ~ 3.871s|
+(RocksDB using `while` iteration to simulate `batch_get`)