add account to rollup node and index node (#494)

* feat: add doc store

* feat: add key store

* feat: add retry to subscribe the message

* feat: add setup rollup

* feat: update cicd

* feat: add docker build

* feat: add test latest

* feat: add a standalone docker build

* feat: add docker files

* fix: update the docker cd name

* feat: add indexer system status

* feat: add setup function

* feat: add setup with permission control

* feat: add meta store client

* feat: update cicd

* fix: update cicd

* fix: fix the typo

.github/workflows/cd.yml

@@ -36,14 +36,18 @@ jobs:
                   override: true
 
             - uses: Swatinem/rust-cache@v2
-            - name: Setup test env
+            - name: Setup build env
               run: |
+                  ROOT_DIR=`pwd`
+                  cd ${ROOT_DIR}/metadata && yarn install && npx hardhat test
+                  ls ${ROOT_DIR}/metadata/artifacts/contracts/DB3MetaStore.sol/
                   if [[ ${{ matrix.job.target }} == x86_64-unknown-linux-gnu ]]; then
                     sudo apt-get install protobuf-compiler
                   fi
                   if [[ ${{ matrix.job.target }} == x86_64-apple-darwin ]]; then
                     brew install protobuf
                   fi
+
             - name: Cargo build
               uses: actions-rs/cargo@v1
               with:
@@ -55,22 +59,9 @@ jobs:
               shell: bash
               run: |
                   cd target/${{ matrix.job.target }}/release
-                  if [[ ${{ matrix.job.target }} == x86_64-unknown-linux-gnu ]]; then
-                    wget https://github.com/tendermint/tendermint/releases/download/v0.37.0-rc2/tendermint_0.37.0-rc2_linux_amd64.tar.gz
-                    mv tendermint_0.37.0-rc2_linux_amd64.tar.gz tendermint.tar.gz
-                    tar -zxf tendermint.tar.gz
-                  fi
-                  if [[ ${{ matrix.job.target }} == x86_64-apple-darwin ]]; then
-                    wget https://github.com/tendermint/tendermint/releases/download/v0.37.0-rc2/tendermint_0.37.0-rc2_darwin_amd64.tar.gz
-                    mv tendermint_0.37.0-rc2_darwin_amd64.tar.gz tendermint.tar.gz
-                    tar -zxf tendermint.tar.gz
-                  fi
                   RELEASE_NAME=db3-${GITHUB_REF/refs\/tags\//}-${{ matrix.job.os-name }}-${{ matrix.job.architecture }}
                   mkdir -p $RELEASE_NAME/bin
-                  mkdir -p $RELEASE_NAME/tm
                   DB3_BINARY_NAME=db3
-                  TM_BINARY_NAME=tendermint
-                  ./tendermint init --home $RELEASE_NAME/tm/
                   mv $DB3_BINARY_NAME $TM_BINARY_NAME $RELEASE_NAME/bin/
                   tar czvf $RELEASE_NAME.tar.gz $RELEASE_NAME
                   ########## create sha256 ##########

.github/workflows/ci.yml

@@ -47,32 +47,25 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - uses: taiki-e/install-action@cargo-llvm-cov
       - uses: taiki-e/install-action@nextest
-      - name: Setup test env
+      - name: Setup Build env
         run: |
           sudo apt-get install protobuf-compiler 
           protoc --version
-          cargo build
-          cd tools
+          ROOT_DIR=`pwd`
+          cd ${ROOT_DIR}/metadata && yarn install && npx hardhat test
+          ls ${ROOT_DIR}/metadata/artifacts/contracts/DB3MetaStore.sol/
+          cd ${ROOT_DIR} && cargo build
+          cd ${ROOT_DIR}/tools
           wget https://github.com/tendermint/tendermint/releases/download/v0.37.0-rc2/tendermint_0.37.0-rc2_linux_amd64.tar.gz
           tar -zxf tendermint_0.37.0-rc2_linux_amd64.tar.gz
           sh start_localnet.sh >tm.log 2>&1 &
           sleep 3
-          ../target/debug/db3 client new-db
-          sleep 1
-          ../target/debug/db3 client show-state
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov nextest --workspace --exclude merkdb  --lcov --output-path lcov.info --no-fail-fast
-      - uses: actions/upload-artifact@v3
-        if: failure()
-        with:
-          name: db3-log-artifact
-          path: |
-            tools/db3.log
-            tools/indexer.log
-            tools/tm.log
       - uses: codecov/codecov-action@v3
         with:
           token: ${{secrets.COV_TOKEN}}
           directory: .
           flags: rust
+

.github/workflows/docker_cd.yml

@@ -0,0 +1,49 @@
+name: CI # Continuous Integration
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+
+  docker_image:
+    name: build_pr_image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - name: Install Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          profile: minimal
+          override: true
+          components: rustfmt
+      - uses: Swatinem/rust-cache@v2
+      - uses: taiki-e/install-action@cargo-llvm-cov
+      - uses: taiki-e/install-action@nextest
+      - name: Setup build env
+        run: |
+          sudo apt-get install protobuf-compiler 
+          protoc --version
+          ROOT_DIR=`pwd`
+          cd ${ROOT_DIR}/metadata && yarn install && npx hardhat compile
+          ls ${ROOT_DIR}/metadata/artifacts/contracts/DB3MetaStore.sol/
+          cd ${ROOT_DIR} && cargo build
+          cp ${ROOT_DIR}/target/debug/db3 ${ROOT_DIR}/docker/
+      - name: docker login
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build Docker image
+        run: |
+          git_hash=$(git rev-parse --short "$GITHUB_SHA")
+          cd docker && docker build . -t ghcr.io/dbpunk-labs/db3:${git_hash}
+          docker push ghcr.io/dbpunk-labs/db3:${git_hash}
+          echo "the image is ghcr.io/dbpunk-labs/db3:${git_hash}"

Cargo.toml

@@ -18,6 +18,6 @@ fastcrypto = {git = "https://github.com/MystenLabs/fastcrypto", rev = "306465d4f
 ethers = {git="https://github.com/imotai/ethers-rs", rev="d526191b7972e8cf4412fee8b71cbf42e0ce7995"}
 tonic = {git="https://github.com/hyperium/tonic", rev="ae7580160431cd25c1eecda4c85014ef6ce8d93f"}
 tonic-web = {git="https://github.com/hyperium/tonic", rev="ae7580160431cd25c1eecda4c85014ef6ce8d93f"}
-arweave-rs = {git="https://github.com/imotai/arweave-rs", rev="7ac5027305db833e4d9b62c967309d5df8fba4f2"}
+arweave-rs = {git="https://github.com/imotai/arweave-rs", rev="4bdb736c565dbce0f3ff698f0265e7e781cca1d6"}
 ejdb2rs = {git="https://github.com/imotai/ejdb2rs", rev="c937a085eda043bd854a0ef4d3e7bb63650610d1"}
 serde_json= "1.0"

docker/Dockerfile

@@ -0,0 +1,10 @@
+FROM ubuntu
+RUN apt update && apt install -y curl git
+RUN curl -sL https://deb.nodesource.com/setup_18.x -o /tmp/nodesource_setup.sh
+RUN bash /tmp/nodesource_setup.sh && apt install nodejs
+RUN npm install -g arlocal serve
+ADD db3 /usr/bin/db3
+RUN chmod +x /usr/bin/db3
+ADD ar_miner.sh /usr/bin/ar_miner.sh
+ADD start_localnet.sh /usr/bin/start_localnet.sh
+CMD ["/bin/bash", "/usr/bin/start_localnet.sh"]

docker/ar_miner.sh

@@ -0,0 +1,16 @@
+#! /bin/sh
+#
+# ar_miner.sh
+# Copyright (C) 2023 jackwang <jackwang@jackwang-ub>
+#
+# Distributed under terms of the MIT license.
+#
+
+npx arlocal >arlocal.log 2>&1 &
+sleep 2
+curl http://127.0.0.1:1984/mint/$1/10000000000000
+while true
+do
+    curl http://127.0.0.1:1984/mine
+    sleep 2
+done

docker/start_localnet.sh

@@ -0,0 +1,19 @@
+#! /bin/bash
+#
+mkdir -p ./mutation_db ./state_db ./doc_db ./keys ./index_meta_db ./index_doc_db
+echo "start store node..."
+/usr/bin/db3 store  --public-host 0.0.0.0 --rollup-interval 30000  > store.log 2>&1 &
+sleep 3
+echo "start index node..."
+/usr/bin/db3 indexer  --public-host 0.0.0.0 > indexer.log 2>&1 &
+sleep 3
+
+AR_ADDRESS=`cat /store.log | grep filestore | awk '{print $NF}'`
+echo "the ar account address ${AR_ADDRESS}"
+echo "start ar testnet ..."
+bash /usr/bin/ar_miner.sh ${AR_ADDRESS} > miner.log 2>&1 &
+sleep 1
+echo "Start the local db3 nodes successfully"
+echo "The storage node url: http://127.0.0.1:26619"
+echo "The index node url: http://127.0.0.1:26639"
+while true; do sleep 10 ; done

metadata/contracts/DB3MetaStore.sol

@@ -25,7 +25,6 @@ contract DB3MetaStore is IDB3MetaStore {
             rollupNodeAddress != address(0),
             "Invalid rollupNodeAddress address"
         );
-
         // Check if network is already registered
         NetworkRegistration storage registration = networkRegistrations[
             networkId
@@ -34,15 +33,13 @@ contract DB3MetaStore is IDB3MetaStore {
             bytes(registration.rollupNodeUrl).length == 0,
             "Network already registered"
         );
-
         // Add new network info to struct and update mapping
         registration.rollupNodeUrl = rollupNodeUrl;
         registration.indexNodeUrls = indexNodeUrls;
         registration.indexNodeAddresses = indexNodeAddresses;
         registration.networkId = networkId;
         registration.admin = msg.sender;
         registration.rollupNodeAddress = rollupNodeAddress;
-
         // Increment registered network counter
         numNetworks++;
     }

metadata/deploy_to_local.sh

@@ -3,4 +3,4 @@
 # deploy_to_scoll.sh
 
 export PRIVATE_KEY=0xad689d9b7751da07b0fb39c5091672cbfe50f59131db015f8a0e76c9790a6fcc
-npx hardhat run --network ganache scripts/deploy.ts
+npx hardhat run --network localhost scripts/deploy.ts

metadata/hardhat.config.ts

@@ -4,17 +4,22 @@ import "@nomicfoundation/hardhat-toolbox";
 const config: HardhatUserConfig = {
   solidity: "0.8.17",
   networks: {
+    mumbai: {
+      url: "https://polygon-mumbai.infura.io/v3/4458cf4d1689497b9a38b1d6bbf05e78" || "",
+      accounts:
+        process.env.PRIVATE_KEY !== undefined ? [process.env.PRIVATE_KEY] : [],
+    },
     scrollPrealpha: {
       url: "https://prealpha-rpc.scroll.io/l2" || "",
       accounts:
         process.env.PRIVATE_KEY !== undefined ? [process.env.PRIVATE_KEY] : [],
     },
-    ganache: {
+
+    localhost: {
       url: "http://127.0.0.1:8545" || "",
       accounts:
         process.env.PRIVATE_KEY !== undefined ? [process.env.PRIVATE_KEY] : [],
     },
-
   }
 };
 

metadata/package.json

@@ -19,6 +19,7 @@
     "chai": "^4.2.0",
     "ethers": "^5.4.7",
     "hardhat": "^2.12.7",
+    "hardhat-abi-exporter": "^2.10.1",
     "hardhat-gas-reporter": "^1.0.8",
     "prettier": "^2.8.4",
     "prettier-plugin-solidity": "^1.1.2",

metadata/scripts/deploy.ts

@@ -1,11 +1,9 @@
 import { ethers } from "hardhat";
 
 async function main() {
-  const Token = await ethers.getContractFactory("Db3Token");
-  const token = await Token.deploy();
-  const Rollup = await ethers.getContractFactory("DB3Rollup");
-  const rollup = await Rollup.deploy(token.address);
-  console.log(`token address ${token.address} rollup address ${rollup.address}`);
+  const DB3MetaStore = await ethers.getContractFactory("DB3MetaStore");
+  const store = await DB3MetaStore.deploy();
+  console.log(`store address ${store.address}`);
 }
 
 // We recommend this pattern to be able to use async/await everywhere

metadata/test/db3_metastroe_test.ts

@@ -433,4 +433,4 @@ describe("DB3MetaStore", function () {
 
 
         });
-
+