Cleanup CI (#5040) (#5047)

* Cleanup CI (#5040)

* Update job name

* Split out examples

Author: tustvold

.github/actions/setup-builder/action.yaml

@@ -32,8 +32,17 @@ runs:
         apt-get install -y protobuf-compiler
     - name: Setup Rust toolchain
       shell: bash
+      # rustfmt is needed for the substrait build script
       run: |
         echo "Installing ${{ inputs.rust-version }}"
         rustup toolchain install ${{ inputs.rust-version }}
         rustup default ${{ inputs.rust-version }}
         rustup component add rustfmt
+    - name: Disable debuginfo generation
+      # Disable full debug symbol generation to speed up CI build and keep memory down
+      # "1" means line tables only, which is useful for panic tracebacks.
+      shell: bash
+      run: echo "RUSTFLAGS=-C debuginfo=1" >> $GITHUB_ENV
+    - name: Enable backtraces
+      shell: bash
+      run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV

.github/workflows/cancel.yml

@@ -18,6 +18,10 @@
 name: Dev
 on: [push, pull_request]
 
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   rat:
     name: Release Audit Tool (RAT)

.github/workflows/dev.yml

@@ -17,6 +17,10 @@
 
 name: Labeler
 
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 on:
   pull_request_target:
     types:

.github/workflows/dev_pr.yml

@@ -17,6 +17,10 @@
 
 name: Rust
 
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 on:
   # always trigger on PR
   push:
@@ -32,10 +36,6 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
       - name: Cache Cargo
@@ -49,18 +49,17 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
-      - name: Check workspace in debug mode
-        run: |
-          cargo check
-      - name: Check workspace in release mode
-        run: |
-          cargo check --release
+
       - name: Check workspace without default features
-        run: |
-          cargo check --no-default-features -p datafusion
+        run: cargo check --no-default-features -p datafusion
+
+      - name: Check workspace in debug mode
+        run: cargo check
+
+      # Note: this does not include dictionary_expressions to reduce codegen
       - name: Check workspace with all features
-        run: |
-          cargo check --workspace --benches --features avro,jit,scheduler,json
+        run: cargo check --workspace --benches --features avro,jit,scheduler,json
+
       - name: Check Cargo.lock for datafusion-cli
         run: |
           # If this test fails, try running `cargo update` in the `datafusion-cli` directory
@@ -70,14 +69,10 @@ jobs:
   # test the crate
   linux-test:
     name: cargo test (amd64)
-    needs: [linux-build-lib]
+    needs: [ linux-build-lib ]
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
         with:
@@ -92,17 +87,27 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
-      - name: Build tests
-        run: |
-          export PATH=$PATH:$HOME/d/protoc/bin
-          cargo test --features avro,jit,scheduler,json,dictionary_expressions --no-run
-      - name: Run tests
-        run: |
-          export PATH=$PATH:$HOME/d/protoc/bin
-          cargo test --features avro,jit,scheduler,json,dictionary_expressions
+      - name: Run tests (excluding doctests)
+        run: cargo test --lib --tests --bins --features avro,jit,scheduler,json,dictionary_expressions
+      - name: Verify Working Directory Clean
+        run: git diff --exit-code
+
+  linux-test-example:
+    name: cargo examples (amd64)
+    needs: [ linux-build-lib ]
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
       - name: Run examples
         run: |
-          export PATH=$PATH:$HOME/d/protoc/bin
           # test datafusion-sql examples
           cargo run --example sql
           # test datafusion-examples
@@ -122,17 +127,34 @@ jobs:
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
+  # Run doc tests
+  linux-test-doc:
+    name: cargo doctest (amd64)
+    needs: [ linux-build-lib ]
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+      # Note: this does not include dictionary_expressions to reduce codegen
+      - name: Run doctests
+        run: cargo test --doc --features avro,jit,scheduler,json
+      - name: Verify Working Directory Clean
+        run: git diff --exit-code
+
   # verify that the benchmark queries return the correct results
   verify-benchmark-results:
     name: verify benchmark results (amd64)
-    needs: [linux-build-lib]
+    needs: [ linux-build-lib ]
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
         with:
@@ -166,7 +188,7 @@ jobs:
 
   integration-test:
     name: "Compare to postgres"
-    needs: [linux-build-lib]
+    needs: [ linux-build-lib ]
     runs-on: ubuntu-latest
     services:
       postgres:
@@ -230,7 +252,7 @@ jobs:
 
   sqllogictest-postgres:
     name: "Run sqllogictest with Postgres runner"
-    needs: [linux-build-lib]
+    needs: [ linux-build-lib ]
     runs-on: ubuntu-latest
     services:
       postgres:
@@ -327,14 +349,10 @@ jobs:
 
   test-datafusion-pyarrow:
     name: cargo test pyarrow (amd64)
-    needs: [linux-build-lib]
+    needs: [ linux-build-lib ]
     runs-on: ubuntu-20.04
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
         with:
@@ -356,10 +374,9 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
-      - name: Run tests
-        run: |
-          cd datafusion
-          cargo test --features=pyarrow
+      - name: Run datafusion-common tests
+        run: cargo test -p datafusion-common --features=pyarrow
+
 
   check-fmt:
     name: Check cargo fmt
@@ -368,11 +385,10 @@ jobs:
       image: amd64/rust
     steps:
       - uses: actions/checkout@v3
-      - name: Setup toolchain
-        run: |
-          rustup toolchain install stable
-          rustup default stable
-          rustup component add rustfmt
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
       - name: Run
         run: |
           echo '' > datafusion/proto/src/generated/datafusion.rs
@@ -422,14 +438,10 @@ jobs:
 
   clippy:
     name: clippy
-    needs: [linux-build-lib]
+    needs: [ linux-build-lib ]
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
         with:
@@ -445,22 +457,17 @@ jobs:
         with:
           rust-version: stable
       - name: Install Clippy
-        run: |
-          rustup component add clippy
+        run: rustup component add clippy
       - name: Run clippy
         run: ci/scripts/rust_clippy.sh
 
   # Check answers are correct when hash values collide
   hash-collisions:
     name: cargo test hash collisions (amd64)
-    needs: [linux-build-lib]
+    needs: [ linux-build-lib ]
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
         with:
@@ -478,19 +485,14 @@ jobs:
       - name: Run tests
         run: |
           cd datafusion
-          # Force all hash values to collide
-          cargo test --all --features=force_hash_collisions
+          cargo test --lib --tests --features=force_hash_collisions
 
   cargo-toml-formatting-checks:
     name: check Cargo.toml formatting
-    needs: [linux-build-lib]
+    needs: [ linux-build-lib ]
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
         with:
@@ -506,8 +508,8 @@ jobs:
         with:
           rust-version: stable
       - name: Install cargo-tomlfmt
-        run: |
-          which cargo-tomlfmt || cargo install cargo-tomlfmt
+        run: which cargo-tomlfmt || cargo install cargo-tomlfmt
+
       - name: Check Cargo.toml formatting
         run: |
           # if you encounter error, try rerun the command below, finally run 'git diff' to
@@ -519,14 +521,10 @@ jobs:
 
   config-docs-check:
     name: check configs.md is up-to-date
-    needs: [linux-build-lib]
+    needs: [ linux-build-lib ]
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
         with: