Merge branch 'datafuselabs:main' into main

.cargo/audit.toml

@@ -1,9 +1,22 @@
 [advisories]
 ignore = [
-    # https://github.com/datafuselabs/databend/issues/2565
-    # need to fix upstream rusoto dependencies on credential and sts for this issue as well
-    "RUSTSEC-2020-0159",
+    # rio: v0.9.4 https://rustsec.org/advisories/RUSTSEC-2020-0021
+    # allows a use-after-free buffer access when a future is leaked
+    "RUSTSEC-2020-0021",
+    # time: Potential segfault in the time crate
+    # We are not affected by this CVE.
+    # And there is no actions we can take, waiting for upstream.
     "RUSTSEC-2020-0071",
-    # https://github.com/datafuselabs/databend/issues/2690
-    "RUSTSEC-2021-0122",
+    # hyper
+    # Lenient `hyper` header parsing of `Content-Length` could allow request smuggling
+    "RUSTSEC-2021-0078",
+    # hyper
+    # Integer overflow in `hyper`'s parsing of the `Transfer-Encoding` header leads to data loss
+    "RUSTSEC-2021-0079",
+    # ansi_term
+    # ansi_term is Unmaintained
+    "RUSTSEC-2021-0139",
+    # buf_redux
+    # buf_redux is Unmaintained
+    "RUSTSEC-2023-0028"
 ]

.cargo/config.toml

@@ -0,0 +1,7 @@
+[env]
+# https://android.googlesource.com/platform/external/jemalloc_new/+/refs/heads/master/TUNING.md
+JEMALLOC_SYS_WITH_MALLOC_CONF = "percpu_arena:percpu,oversize_threshold:0,background_thread:true,dirty_decay_ms:5000,muzzy_decay_ms:5000"
+
+## If you are using aarch64-apple-darwin, you may need to set the following:
+## JEMALLOC_SYS_WITH_LG_PAGE = "14"
+## JEMALLOC_SYS_WITH_MALLOC_CONF = "oversize_threshold:0,dirty_decay_ms:5000,muzzy_decay_ms:5000"

.config/nextest.toml

@@ -0,0 +1,8 @@
+[test-groups]
+serial-integration = { max-threads = 1 }
+
+
+[[profile.default.overrides]]
+filter = 'package(databend-meta)'
+test-group = 'serial-integration'
+

.devcontainer/Dockerfile

@@ -0,0 +1,17 @@
+FROM datafuselabs/build-tool:dev
+
+ENV TERM xterm-256color
+
+RUN apt-get update && apt-get install -y git ripgrep && \
+    curl -sSo /tmp/install.sh https://starship.rs/install.sh && \
+    chmod +x /tmp/install.sh && \
+    /tmp/install.sh --yes && \
+    printf 'eval "$(starship init bash)"' >> /etc/bash.bashrc && \
+    rm -rf /tmp/* /var/lib/apt/lists/*
+
+RUN useradd -u 1000 builder
+RUN mkdir /home/builder && chown -R builder /home/builder && \
+    usermod --shell /bin/bash builder
+RUN printf "builder ALL=(ALL:ALL) NOPASSWD:ALL\n" > /etc/sudoers.d/databend
+
+USER builder

.devcontainer/devcontainer.json

@@ -0,0 +1,27 @@
+{
+	"name": "Databend",
+	"mounts": [
+		"source=${localEnv:HOME}/.cargo/registry,target=/opt/rust/cargo/registry,type=bind,consistency=cached",
+		"source=${localEnv:HOME}/.cargo/git,target=/opt/rust/cargo/git,type=bind,consistency=cached",
+		"source=${localEnv:HOME},target=/home/builder,type=bind,consistency=cached"
+	],
+	"dockerFile": "Dockerfile",
+	"remoteUser": "builder",
+	"updateRemoteUserUID": true,
+	"postCreateCommand": "sudo chown -R builder /opt/rust",
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"rust-lang.rust-analyzer",
+				"serayuzgur.crates",
+				"vadimcn.vscode-lldb"
+			],
+			"settings": {
+				"editor.formatOnSave": true,
+				"files.exclude": {
+					"**/LICENSE": true
+				}
+			}
+		}
+	}
+}

.editorconfig

@@ -0,0 +1,14 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+# Make sure all code is in utf-8 charset
+[*.{rs,js,toml,py,yml}]
+charset = utf-8

.gitattributes

@@ -1,3 +1,2 @@
 docker/** -text
-scripts/** -text
-website/databend/** -text
+scripts/** -text

.github/CODEOWNERS

@@ -1,43 +1,45 @@
-# Global rule
-*    @BohuTANG
-
-# Query rule
-/query/                  @datafuselabs/query-members
-/common/array/           @datafuselabs/query-members
-/common/cache/           @datafuselabs/query-members
-/common/clickhouse-srv/  @datafuselabs/query-members
-/common/context/         @datafuselabs/query-members
-/common/datablocks/      @datafuselabs/query-members
-/common/datavalues/      @datafuselabs/query-members
-/common/flight-rpc/      @datafuselabs/query-members
-/common/functions/       @datafuselabs/query-members
-/common/infallible/      @datafuselabs/query-members
-/common/io/              @datafuselabs/query-members
-/common/management/      @datafuselabs/query-members
-/common/mem-allocator/   @datafuselabs/query-members
-/common/metrics/         @datafuselabs/query-members
-/common/planners/        @datafuselabs/query-members
-/common/streams/         @datafuselabs/query-members
-/common/tracing/         @datafuselabs/query-members
-/common/base/            @datafuselabs/query-members
-/common/building/        @datafuselabs/query-members
-
-# Storage rule
-/common/dal/             @datafuselabs/storage-members
-/query/src/datasources/  @datafuselabs/storage-members
-
-# Meta rule
-/metasrv                 @datafuselabs/meta-members
-/kvsrv                   @datafuselabs/meta-members
-/common/meta/            @datafuselabs/meta-members
-/common/raft-store/      @datafuselabs/meta-members
-/common/sled-store/      @datafuselabs/meta-members
-
-# Cli rule
-/docker                  @datafuselabs/cli-members
-/cli                     @datafuselabs/cli-members
-
-# Tests rule
-
-# Website rule
-/website  @datafuselabs/doc-members
+# Query
+/src/common/arrow/         @datafuselabs/databend-query-reviewer
+/src/common/hashtable/     @datafuselabs/databend-query-reviewer
+/src/common/io/            @datafuselabs/databend-query-reviewer
+/src/query/datablocks/     @datafuselabs/databend-query-reviewer
+/src/query/datavalues/     @datafuselabs/databend-query-reviewer
+/src/query/functions/      @datafuselabs/databend-query-reviewer
+/src/query/management/     @datafuselabs/databend-query-reviewer
+/src/query/streams/        @datafuselabs/databend-query-reviewer
+
+# Executor
+/src/common/base/                  @datafuselabs/databend-executor-reviewer
+/src/common/exception/             @datafuselabs/databend-executor-reviewer
+/src/query/planners/               @datafuselabs/databend-executor-reviewer
+/src/query/service/src/clusters/   @datafuselabs/databend-executor-reviewer
+/src/query/service/src/pipelines/  @datafuselabs/databend-executor-reviewer
+/src/query/service/src/api/rpc/    @datafuselabs/databend-executor-reviewer
+
+# Planner
+/src/query/ast/                       @datafuselabs/databend-planner-reviewer
+/src/query/service/src/optimizers/    @datafuselabs/databend-planner-reviewer
+
+# MetaSrv
+/src/common/grpc/         @datafuselabs/databend-meta-reviewer
+/src/meta/                @datafuselabs/databend-meta-reviewer
+/src/meta/protos/proto    @drmingdrmer
+
+# Storage
+/src/common/cache/                 @datafuselabs/databend-storage-reviewer
+/src/common/contexts/              @datafuselabs/databend-storage-reviewer
+/src/common/storage/               @datafuselabs/databend-storage-reviewer
+/src/query/catalog/                @datafuselabs/databend-storage-reviewer
+/src/query/storages/               @datafuselabs/databend-storage-reviewer
+/src/query/service/src/catalogs/   @datafuselabs/databend-storage-reviewer
+/src/query/service/src/storages/   @datafuselabs/databend-storage-reviewer
+
+
+# API and handler
+/src/common/http/                  @datafuselabs/databend-api-reviewer
+/src/query/service/src/servers/    @datafuselabs/databend-api-reviewer
+
+# Tool
+/docker/               @datafuselabs/databend-tool-reviewer
+/scripts/              @datafuselabs/databend-tool-reviewer
+/.github/              @datafuselabs/databend-tool-reviewer

.github/ISSUE_TEMPLATE/20_bug_report.yml

@@ -41,12 +41,6 @@ body:
     validations:
       required: true
 
-  - type: textarea
-    attributes:
-      label: What You Expected?
-    validations:
-      required: true
-
   - type: textarea
     attributes:
       label: How to Reproduce?
@@ -60,10 +54,6 @@ body:
 
         If it is hard to reproduce, please also explain the general scene.
 
-  - type: textarea
-    attributes:
-      label: Anything Else?
-
 
   - type: checkboxes
     attributes:

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,25 +4,4 @@ I hereby agree to the terms of the CLA available at: https://databend.rs/dev/pol
 
 Summary about this PR
 
-## Changelog
-
-- New Feature
-- Bug Fix
-- Improvement
-- Performance Improvement
-- Build/Testing/CI
-- Need Documentation (Need to add documentation to https://databend.rs)
-- Documentation
-- Other 
-- Not for changelog (changelog entry is not required)
-
-## Related Issues
-
-Fixes #issue
-
-## Test Plan
-
-Unit Tests
-
-Stateless Tests
-
+- Closes #issue

.github/actions-rs/grcov.yml

@@ -18,4 +18,3 @@ ignore:
   - "*/tracing/*"
   - "*/api/http/debug/*"
   - "*/datasources/example/*"
-  - "cli/**/*"

.github/actions/artifact_download/action.yml

@@ -0,0 +1,101 @@
+name: "Download Artifacts"
+description: "Download build Result Binaries"
+inputs:
+  sha:
+    description: "git sha"
+    required: true
+  target:
+    description: ""
+    required: true
+  category:
+    description: "default/hdfs"
+    required: false
+    default: default
+  path:
+    description: "default to ./target/${BUILD_PROFILE}/"
+    required: false
+    default: ""
+  artifacts:
+    description: "Artifacts to download, only works with s3"
+    required: false
+    default: "meta,query"
+outputs:
+  path:
+    description: ""
+    value: ${{ steps.info.outputs.path }}
+runs:
+  using: "composite"
+  steps:
+    - name: Check Disk Space
+      shell: bash
+      run: |
+        df -h
+
+    - name: Get Download Info
+      id: info
+      shell: bash
+      run: |
+        if [[ -z "${{ inputs.path }}" ]]; then
+          path="./target/${{ env.BUILD_PROFILE }}"
+        else
+          path="${{ inputs.path }}"
+        fi
+        echo "path=${path}" >> $GITHUB_OUTPUT
+
+    - uses: actions/download-artifact@v3
+      if: env.RUNNER_PROVIDER == 'github'
+      with:
+        name: ${{ env.BUILD_PROFILE }}-${{ inputs.sha }}-${{ inputs.target }}-${{ inputs.category }}
+        path: ${{ steps.info.outputs.path }}/
+
+    - name: Make Executable
+      if: env.RUNNER_PROVIDER == 'github'
+      shell: bash
+      run: |
+        artifacts="${{ inputs.artifacts }}"
+        for artifact in ${artifacts//,/ }; do
+          if [[ "${artifact}" == "open-sharing" ]]; then
+            chmod +x ${{ steps.info.outputs.path }}/open-sharing || true
+          else
+            chmod +x ${{ steps.info.outputs.path }}/databend-$artifact
+          fi
+        done
+
+    - name: Download artifact from s3
+      if: env.RUNNER_PROVIDER == 'aws'
+      shell: bash
+      run: |
+        artifacts="${{ inputs.artifacts }}"
+        for artifact in ${artifacts//,/ }; do
+          if [[ "${artifact}" == "open-sharing" ]]; then
+            echo "==> fetching open-sharing ..."
+            aws s3 cp s3://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/open-sharing ${{ steps.info.outputs.path }}/open-sharing --no-progress || true
+            chmod +x ${{ steps.info.outputs.path }}/open-sharing || true
+          else
+            echo "==> fetching databend-$artifact ..."
+            aws s3 cp s3://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/databend-$artifact ${{ steps.info.outputs.path }}/databend-$artifact --no-progress
+            chmod +x ${{ steps.info.outputs.path }}/databend-$artifact
+          fi
+        done
+
+    - name: Download artifact from gcs
+      if: env.RUNNER_PROVIDER == 'gcp'
+      shell: bash
+      run: |
+        artifacts="${{ inputs.artifacts }}"
+        for artifact in ${artifacts//,/ }; do
+          if [[ "${artifact}" == "open-sharing" ]]; then
+            echo "==> fetching open-sharing ..."
+            gcloud storage cp gs://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/open-sharing ${{ steps.info.outputs.path }}/open-sharing || true
+            chmod +x ${{ steps.info.outputs.path }}/open-sharing || true
+          else
+            echo "==> fetching databend-$artifact ..."
+            gcloud storage cp gs://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/databend-$artifact ${{ steps.info.outputs.path }}/databend-$artifact
+            chmod +x ${{ steps.info.outputs.path }}/databend-$artifact
+          fi
+        done
+
+    - name: Check Disk Space
+      shell: bash
+      run: |
+        df -h

.github/actions/artifact_failure/action.yml

@@ -0,0 +1,27 @@
+name: "Upload failure Artifacts"
+description: "Upload failure Artifacts"
+inputs:
+  name:
+    description: ""
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: pack failure artifacts
+      shell: bash
+      run: |
+        killall -9 databend-query || true
+        killall -9 databend-meta || true
+
+        docker ps -a
+        mkdir -p .databend/docker/
+        docker ps -a --format "{{.Names}}" | while read -r line; do
+            docker logs "$line" > .databend/docker/"$line".log
+        done
+
+        mkdir -p target
+        find -type d -name .databend -print0 | xargs -0 tar -zcf target/failure-${{ inputs.name }}.tar.gz
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.name }}
+        path: target/failure-${{ inputs.name }}.tar.gz