chore: add integration test for cluster mode.

.github/workflows/ci.yml

@@ -28,6 +28,8 @@ env:
   RUSTFLAGS: "-C debuginfo=1"
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: "1"
+  CERESMETA_IMAGE_NAME: ceresdb/ceresmeta-server:latest
+  CERESDB_IMAGE_NAME: ceresdb/ceresdb-server:latest
   LOCK_FILE: Cargo.lock
 
 jobs:
@@ -37,7 +39,7 @@ jobs:
     timeout-minutes: 60
     strategy:
       matrix:
-        rust: [nightly-2022-08-08]
+        rust: [ nightly-2022-08-08 ]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -90,7 +92,7 @@ jobs:
     timeout-minutes: 60
     strategy:
       matrix:
-        rust: [nightly-2022-08-08]
+        rust: [ nightly-2022-08-08 ]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -134,18 +136,22 @@ jobs:
     timeout-minutes: 60
     strategy:
       matrix:
-        rust: [nightly-2022-08-08]
+        rust: [ nightly-2022-08-08 ]
     steps:
+      - name: Download CeresMeta Code
+        uses: actions/checkout@v3
+        with:
+          repository: CeresDB/ceresmeta
+          ref: refs/heads/main
+      - name: Build CeresMeta Docker image
+        run: docker build -t ${CERESMETA_IMAGE_NAME} .
+      - name: Add CeresDB Hosts
+        run: |
+          sudo echo "127.0.0.1 ceresdb0" | sudo tee -a /etc/hosts
+          sudo echo "127.0.0.1 ceresdb1" | sudo tee -a /etc/hosts
       - uses: actions/checkout@v3
         with:
           submodules: true
-      - run: |
-          rustup set auto-self-update disable
-          rustup toolchain install ${{ matrix.rust }} --profile minimal
-      - name: Release Disk Quota
-        run: |
-          sudo rm -rf /usr/local/lib/android # release about 10 GB
-          sudo rm -rf /usr/share/dotnet # release about 20GB
       - name: Cache Rust Dependencies
         uses: actions/cache@v3
         with:
@@ -157,12 +163,35 @@ jobs:
             debug-${{ runner.os }}-${{ hashFiles('rust-toolchain') }}-
             debug-${{ runner.os }}-
             debug-
+      - run: |
+          rustup set auto-self-update disable
+          rustup toolchain install ${{ matrix.rust }} --profile minimal
+      - name: Release Disk Quota
+        run: |
+          sudo rm -rf /usr/local/lib/android # release about 10 GB
+          sudo rm -rf /usr/share/dotnet # release about 20GB
       - name: Setup Build Environment
         run: |
           sudo apt update
           sudo apt install --yes protobuf-compiler
+      - name: Build CeresDB
+        run: |
+          make build-debug
+      - name: Copy Target
+        run: |
+          cp ./target/debug/ceresdb-server ./ceresdb-server
+      - name: Build CeresDB Docker Image
+        run: docker build -f ./Dockerfile-CI -t ${CERESDB_IMAGE_NAME} .
       - name: Run black-box tests
         run: |
           make test-black
         env:
           RUST_BACKTRACE: "1"
+      - name: Upload Logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: CI
+          path: |
+            /tmp/ceresdb-stdout.log
+            /tmp/ceresdb-stderr.log

Dockerfile-CI

@@ -0,0 +1,25 @@
+## CeresDB
+FROM ubuntu:22.04
+
+RUN useradd -m -s /bin/bash ceres
+
+RUN apt update && \
+    apt install --yes curl gdb iotop cron vim less net-tools && \
+    apt clean \
+
+RUN apt update && apt install --yes libssl-dev
+
+ENV RUST_BACKTRACE 1
+
+COPY ./ceresdb-server /usr/bin/ceresdb-server
+RUN chmod +x /usr/bin/ceresdb-server
+
+COPY ./docker/entrypoint.sh /entrypoint.sh
+COPY ./docs/minimal.toml /etc/ceresdb/ceresdb.toml
+
+COPY ./docker/tini /tini
+RUN chmod +x /tini
+
+ARG USER ceres
+
+ENTRYPOINT ["/tini", "--", "/entrypoint.sh"]

Makefile

@@ -6,6 +6,10 @@ init:
 	echo "init"
 	echo "Git branch: $GITBRANCH"
 
+build-debug:
+	ls -alh
+	cd $(DIR); cargo build
+
 build:
 	ls -alh
 	cd $(DIR); cargo build --release

docker-compose.yaml

@@ -0,0 +1,31 @@
+version: '2'
+
+services:
+  ceresmeta:
+    image: ceresdb/ceresmeta-server:latest
+    ports:
+      - "2379:2379"
+    volumes:
+      - ./docker/ceresmeta.toml:/etc/ceresmeta/ceresmeta.toml
+
+  ceresdb0:
+    image: ceresdb/ceresdb-server:latest
+    restart: always
+    ports:
+      - "8832:8832"
+      - "5441:5441"
+    volumes:
+      - ./docker/ceresdb-cluster-0.toml:/etc/ceresdb/ceresdb.toml
+    depends_on:
+      - ceresmeta
+
+  ceresdb1:
+    image: ceresdb/ceresdb-server:latest
+    restart: always
+    ports:
+      - "8833:8833"
+      - "5442:5442"
+    volumes:
+      - ./docker/ceresdb-cluster-1.toml:/etc/ceresdb/ceresdb.toml
+    depends_on:
+      - ceresmeta

docker/ceresdb-cluster-0.toml

@@ -0,0 +1,40 @@
+[node]
+addr = "ceresdb0"
+
+[server]
+bind_addr = "0.0.0.0"
+http_port = 5441
+grpc_port = 8832
+mysql_port = 3307
+log_level = "info"
+deploy_mode = "Cluster"
+
+[server.forward]
+enable = true
+
+[analytic.storage]
+mem_cache_capacity = '1G'
+mem_cache_partition_bits = 0
+
+[analytic.storage.object_store]
+type = "Local"
+data_dir = "/tmp/ceresdb0"
+
+[analytic.wal_storage]
+type = "RocksDB"
+data_dir = "/tmp/ceresdb0"
+
+[cluster_deployment]
+mode = "WithMeta"
+cmd_channel_buffer_size = 10
+
+[cluster_deployment.meta_client]
+# Only support "defaultCluster" currently.
+cluster_name = "defaultCluster"
+meta_addr = "http://ceresmeta:2379"
+lease = "10s"
+timeout = "5s"
+
+[limiter]
+write_block_list = ['mytable1']
+read_block_list = ['mytable1']

docker/ceresdb-cluster-1.toml

@@ -0,0 +1,40 @@
+[node]
+addr = "ceresdb1"
+
+[server]
+bind_addr = "0.0.0.0"
+http_port = 5442
+grpc_port = 8833
+mysql_port = 3307
+log_level = "info"
+deploy_mode = "Cluster"
+
+[server.forward]
+enable = true
+
+[analytic.storage]
+mem_cache_capacity = '1G'
+mem_cache_partition_bits = 0
+
+[analytic.storage.object_store]
+type = "Local"
+data_dir = "/tmp/ceresdb0"
+
+[analytic.wal_storage]
+type = "RocksDB"
+data_dir = "/tmp/ceresdb0"
+
+[cluster_deployment]
+mode = "WithMeta"
+cmd_channel_buffer_size = 10
+
+[cluster_deployment.meta_client]
+# Only support "defaultCluster" currently.
+cluster_name = "defaultCluster"
+meta_addr = "http://ceresmeta:2379"
+lease = "10s"
+timeout = "5s"
+
+[limiter]
+write_block_list = ['mytable1']
+read_block_list = ['mytable1']

docker/ceresmeta.toml

@@ -0,0 +1,15 @@
+etcd-start-timeout-ms = 30000
+peer-urls = "http://0.0.0.0:2380"
+advertise-client-urls = "http://0.0.0.0:2379"
+advertise-peer-urls = "http://0.0.0.0:2380"
+client-urls = "http://0.0.0.0:2379"
+data-dir = "/tmp/ceresmeta0"
+node-name = "meta0"
+initial-cluster = "meta0=http://0.0.0.0:2380"
+default-cluster-node-count = 2
+
+[log]
+level = "info"
+
+[etcd-log]
+level = "info"

integration_tests/Cargo.toml

@@ -9,5 +9,7 @@ workspace = true
 anyhow = "1.0.58"
 async-trait = "0.1"
 ceresdb-client-rs = { git = "https://github.com/CeresDB/ceresdb-client-rs.git", rev = "69948b9963597ccdb7c73756473393972dfdebd3" }
-sqlness = "0.3"
+sql = { workspace = true }
+sqlness = { git = "https://github.com/CeresDB/sqlness.git", rev = "e6d602ed7d894ffb806580f17f4a6f1283abbf1f" }
+sqlparser = { workspace = true }
 tokio = { workspace = true }

integration_tests/Makefile

@@ -5,7 +5,8 @@ DATA_DIR = /tmp/ceresdb
 
 export CERESDB_BINARY_PATH ?= $(ROOT)/../target/$(MODE)/ceresdb-server
 export CERESDB_SERVER_ENDPOINT ?= 127.0.0.1:8831
-export CERESDB_TEST_CASE_PATH ?= $(ROOT)/cases
+export CERESDB_CLUSTER_SERVER_ENDPOINT ?= 127.0.0.1:8832
+export CERESDB_TEST_CASE_PATH ?= $(ROOT)/cases/env
 export CERESDB_TEST_BINARY ?= $(ROOT)/../target/$(MODE)/ceresdb-test
 export CERESDB_STDOUT_FILE ?= /tmp/ceresdb-stdout.log
 export CERESDB_STDERR_FILE ?= /tmp/ceresdb-stderr.log