Improve action workflow configuration (#2307)

* Improve action workflow configuration

* Improve again

* Cleanup release related event for package and docker

* Update cache of docker images

Author: yixinglu

.github/workflows/build.yaml

@@ -3,12 +3,6 @@ name: docker
 on:
   schedule:
     - cron: '0 18 * * *'
-  push:
-    branches:
-      - 'v[0-9]+.*'
-    paths-ignore:
-      - 'docs/**'
-      - '**.md'
 
 jobs:
   docker:
@@ -23,24 +17,16 @@ jobs:
           - console
           - graph
     steps:
-      - uses: azure/docker-login@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
       - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Compute tag
-        id: tag
-        run: |
-          [[ "${{ github.event_name }}" == "push" ]] && \
-            echo "::set-output name=tag::$(cut -d/ -f3 <<< ${{ github.ref }})" || \
-            echo "::set-output name=tag::nightly"
-        shell: bash
-      - name: Build image
-        env:
-          IMAGE_NAME: ${{ secrets.DOCKER_USERNAME }}/nebula-${{ matrix.service }}:${{ steps.tag.outputs.tag }}
-        run: |
-          docker build -t ${IMAGE_NAME} -f docker/Dockerfile.${{ matrix.service }} .
-          docker push ${IMAGE_NAME}
-        shell: bash
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: ${{ secrets.DOCKER_USERNAME }}/nebula-${{ matrix.service }}
+          tags: nightly
+          dockerfile: docker/Dockerfile.${{ matrix.service }}
+          cache_froms: vesoft/nebula-dev:centos7,centos:7
+          add_git_labels: true

.github/workflows/docker.yaml

@@ -3,13 +3,10 @@ name: package
 on:
   schedule:
     - cron: '0 18 * * *'
-  push:
-    # Releases are all branch based, and the branch package here is for testing.
-    branches:
-      - 'v[0-9]+.*'
-    paths-ignore:
-      - 'docs/**'
-      - '**.md'
+
+defaults:
+  run:
+    shell: bash
 
 jobs:
   package:
@@ -28,25 +25,16 @@ jobs:
       - uses: actions/checkout@v1
       - name: Package
         run: ./package/package.sh
-        shell: bash
-      - name: Compute version
-        id: version
-        env:
-          CPACK_OUTPUT_DIR: build/cpack_output
-        run: |
-          [[ "${{ github.event_name }}" == "push" ]] && \
-            echo "::set-output name=version::$(cut -d/ -f3 <<< ${{ github.ref }})" || \
-            (echo "::set-output name=version::nightly"
-              cd $CPACK_OUTPUT_DIR && \
-              filename=$(find . -type f \( -iname \*.deb -o -iname \*.rpm \) -exec basename {} \;)
-              echo "::set-output name=filepath::$CPACK_OUTPUT_DIR/$filename")
-        shell: bash
-      - name: upload oss
-        if: github.event_name != 'push'
+      - name: Upload oss
         run: |
-          ./ci/scripts/upload-oss.sh OSS_ENDPOINT=${{ secrets.OSS_ENDPOINT }} OSS_ID=${{ secrets.OSS_ID }} OSS_SECRET=${{ secrets.OSS_SECRET }} filepath=${{ steps.version.outputs.filepath }} nightly=true
-        shell: bash
+          filepath=$(find build/cpack_output -type f \( -iname \*.deb -o -iname \*.rpm \))
+          ./ci/scripts/upload-oss.sh \
+            OSS_ENDPOINT=${{ secrets.OSS_ENDPOINT }} \
+            OSS_ID=${{ secrets.OSS_ID }} \
+            OSS_SECRET=${{ secrets.OSS_SECRET }} \
+            filepath=$filepath \
+            nightly=true
       - uses: actions/upload-artifact@v1
         with:
-          name: ${{ matrix.os }}-${{ steps.version.outputs.version }}
+          name: ${{ matrix.os }}-nightly
           path: build/cpack_output

.github/workflows/package.yaml

@@ -7,19 +7,25 @@ on:
       - master
       - 'v[0-9]+.*'
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   lint:
     name: cpplint
-    if: contains(join(toJson(github.event.pull_request.labels.*.name)), 'ready-for-testing')
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'ready-for-testing') }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
         with:
-          fetch-depth: 2
+          fetch-depth: 1
+      - uses: jitterbit/get-changed-files@v1
+        id: diff
       - name: Cpplint
         run: |
-          cd .linters/cpp && ln -snf hooks/pre-commit.sh
-          ./pre-commit.sh $(git --no-pager diff --name-only $(git log --oneline -n 1 | cut -d ' ' -f 1,5) | sed -E 's/(.*)/..\/..\/\1/g')
+          ln -snf $PWD/.linters/cpp/hooks/pre-commit.sh $PWD/.linters/cpp/pre-commit.sh
+          .linters/cpp/pre-commit.sh ${{ steps.diff.outputs.all }}
 
   build:
     name: build
@@ -40,62 +46,58 @@ jobs:
     container:
       image: vesoft/nebula-dev:${{ matrix.os }}
       env:
-        CCACHE_DIR: /tmp/ccache/${{ matrix.os }}-${{ matrix.compiler }}
-      options: --mount type=tmpfs,destination=/tmp/ccache,tmpfs-size=1073741824 -v /tmp/ccache/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/${{ matrix.os }}-${{ matrix.compiler }} -v /etc/action/secrets/codecov:/etc/action/secrets/codecov
+        CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
+        CCACHE_MAXSIZE: 1G
+      volumes:
+        - /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
+      options: --mount type=tmpfs,destination=/tmp/ccache/nebula,tmpfs-size=1073741824 --cap-add=SYS_PTRACE
     steps:
       - uses: actions/checkout@v2
         with:
-          fetch-depth: 2
-      - name: Ignore only documents changes
-        id: ignore_docs
-        run: |
-          num_source_files=$(git --no-pager diff --name-only $(git log --oneline -n 1 | cut -d' ' -f1,5) | grep '^src\|^CMakeLists.txt\|^cmake\|^.github/workflows' | wc -l)
-          echo "::set-output name=num_source_files::${num_source_files}"
+          fetch-depth: 1
       - name: Prepare environment
         run: |
-          [ -f "$CCACHE_DIR/ccache.conf" ] || cp ci/ccache.conf "$CCACHE_DIR"
           [ -d build/ ] && rm -rf build/* || mkdir -p build
-        shell: bash
-      - name: CMake with gcc-9.2
-        if: matrix.compiler == 'gcc-9.2' && matrix.os != 'centos7' && steps.ignore_docs.outputs.num_source_files != 0
-        run: |
-          cmake -DCMAKE_BUILD_TYPE=Release -B build/
-      - name: CMake with Coverage
-        if: matrix.compiler == 'gcc-9.2' && matrix.os == 'centos7' && steps.ignore_docs.outputs.num_source_files != 0
+      - name: CMake
         run: |
-          cmake -DENABLE_COVERAGE=ON -B build/
-      - name: CMake with clang-9
-        if: matrix.compiler == 'clang-9' && steps.ignore_docs.outputs.num_source_files != 0
-        run: |
-          cmake \
-            -DCMAKE_CXX_COMPILER=clang++ \
-            -DCMAKE_C_COMPILER=clang \
-            -DENABLE_ASAN=on \
-            -B build/
+          case ${{ matrix.os }} in
+          centos7)
+            # build with coverage
+            cmake -DENABLE_COVERAGE=ON -B build/
+            ;;
+          ubuntu1804)
+            case ${{ matrix.compiler }} in
+            gcc*)
+              # build with release type
+              cmake -DCMAKE_BUILD_TYPE=Release -B build/
+              ;;
+            clang*)
+              cmake \
+                -DCMAKE_CXX_COMPILER=clang++ \
+                -DCMAKE_C_COMPILER=clang \
+                -DENABLE_ASAN=on \
+                -B build/
+              ;;
+            esac
+            ;;
+          esac
       - name: Make
-        if: steps.ignore_docs.outputs.num_source_files != 0
-        run: |
-          cmake --build build/ -j $(nproc)
+        run: cmake --build build/ -j $(($(nproc)/2+1))
       - name: CTest with multiple threads
-        if: steps.ignore_docs.outputs.num_source_files != 0
         timeout-minutes: 15
-        continue-on-error: false
         working-directory: build
         env:
-          ASAN_SYMBOLIZER_PATH: /opt/vesoft/toolset/clang/9.0.0/bin/llvm-symbolizer
           ASAN_OPTIONS: fast_unwind_on_malloc=1
         run: |
           ctest \
-            -j $(($(nproc)/2)) \
+            -j $(($(nproc)/2+1)) \
             --timeout 400 \
             --output-on-failure
-        shell: bash
       - name: CTest with single thread
-        if: failure() && steps.ignore_docs.outputs.num_source_files != 0
+        if: failure()
         timeout-minutes: 30
         working-directory: build
         env:
-          ASAN_SYMBOLIZER_PATH: /opt/vesoft/toolset/clang/9.0.0/bin/llvm-symbolizer
           ASAN_OPTIONS: fast_unwind_on_malloc=0
         run: |
           ctest \
@@ -104,13 +106,12 @@ jobs:
             --rerun-failed
       - name: Testing Coverage Report
         working-directory: build
-        if: success() && matrix.compiler == 'gcc-9.2' && matrix.os == 'centos7' && steps.ignore_docs.outputs.num_source_files != 0
+        if: success() && matrix.compiler == 'gcc-9.2' && matrix.os == 'centos7'
         run: |
           set -e
           fastcov -o coverage.info -l --exclude=scanner.lex
           lcov --remove coverage.info '*/opt/vesoft/*' -o clean.info
-          bash <(curl -s https://codecov.io/bash) -Z -t $(cat /etc/action/secrets/codecov) -f clean.info
-        shell: bash
+          bash <(curl -s https://codecov.io/bash) -Z -f clean.info
       - name: Cleanup
         if: always()
         run: rm -rf build