Tweak ci pipeline (#595)

Author: anakinxc

.circleci/config.yml

@@ -99,11 +99,4 @@ workflows:
           # This is mandatory to trigger a pipeline when pushing a tag
           filters:
             tags:
-              only: /.*(?<!dev\d{8})$/
-      - continuation/continue:
-          configuration_path: .circleci/dev-release-config.yml
-          # This is mandatory to trigger a pipeline when pushing a tag
-          filters:
-            tags:
-              only: /.*(?<=dev\d{8})$/
-
+              only: /.*/

.circleci/continue-config.yml

@@ -21,21 +21,25 @@ parameters:
     type: boolean
     default: false
 
-# Define a job to be invoked later in a workflow.
-# See: https://circleci.com/docs/2.0/configuration-reference/#jobs
-jobs:
-  linux_ut:
-    # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub.
-    # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor
-    docker:
-      - image: secretflow/ubuntu-base-ci:latest
+executors:
+  linux_executor: # declares a reusable executor
     parameters:
       resource_class:
         type: string
+    docker:
+      - image: secretflow/ubuntu-base-ci:latest
     resource_class: << parameters.resource_class >>
     shell: /bin/bash --login -eo pipefail
-    # Add steps to the job
-    # See: https://circleci.com/docs/2.0/configuration-reference/#steps
+
+commands:
+  build_and_test:
+    parameters:
+      targets:
+        type: string
+        default: "//..."
+      extra_bazel_test_args:
+        type: string
+        default: ""
     steps:
       # Kill the whole ci after 1hr
       - run:
@@ -45,14 +49,12 @@ jobs:
             sleep 7200
             echo "Canceling workflow as too much time has elapsed"
             curl -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/cancel?circle-token=${BUILD_TIMER_TOKEN}"
-      - checkout
       - run:
           name: Checkout devtools
           command: git clone https://github.com/secretflow/devtools.git ../devtools
       - run:
           name: "Install dependencies"
           command: |
-            python3 -m pip install -U pip
             python3 -m pip install -r requirements.txt
             python3 -m pip install -r requirements-dev.txt
       - run:
@@ -62,83 +64,57 @@ jobs:
             ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json --min_download
       - run:
           name: "build"
-          command: bazel build //... -c opt --ui_event_filters=-info,-debug,-warning --jobs 20
+          command: bazel build <<parameters.targets>> -c opt --ui_event_filters=-info,-debug,-warning --jobs 20
       - run:
           name: "test"
           command: |
             set +e
             declare -i test_status
-            bazel test //... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors| tee test_result.log; test_status=${PIPESTATUS[0]}
+            bazel test <<parameters.extra_bazel_test_args>> <<parameters.targets>> -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors| tee test_result.log; test_status=${PIPESTATUS[0]}
 
             sh ../devtools/rename-junit-xml.sh
-            find bazel-bin/ -executable -type f -name "*_test"  -print0 | xargs -0 tar -cvzf test_binary.tar.gz
             find bazel-testlogs/ -type f -name "test.log"  -print0 | xargs -0 tar -cvzf test_logs.tar.gz
 
             exit ${test_status}
       - store_test_results:
           path: test-results
-      - store_artifacts:
-          path: test_binary.tar.gz
       - store_artifacts:
           path: test_logs.tar.gz
-  linux_ml_ut:
-    docker:
-      - image: secretflow/ubuntu-base-ci:latest
-    # arm is not supported
-    resource_class: 2xlarge
-    shell: /bin/bash --login -eo pipefail
+
+# Define a job to be invoked later in a workflow.
+# See: https://circleci.com/docs/2.0/configuration-reference/#jobs
+jobs:
+  linux_ut:
+    parameters:
+      resource_class:
+        type: string
+    executor:
+      name: linux_executor
+      resource_class: << parameters.resource_class >>
+    # Add steps to the job
+    # See: https://circleci.com/docs/2.0/configuration-reference/#steps
     steps:
-      - run:
-          name: Cancel build after set time
-          background: true
-          command: |
-            sleep 7200
-            echo "Canceling workflow as too much time has elapsed"
-            curl -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/cancel?circle-token=${BUILD_TIMER_TOKEN}"
       - checkout
+      - build_and_test
+  linux_ml_ut:
+    executor:
+      name: linux_executor
+      resource_class: 2xlarge+
+    steps:
       - run:
-          name: Checkout devtools
-          command: git clone https://github.com/secretflow/devtools.git ../devtools
-      - run:
-          name: "Install dependencies"
+          name: Install extra deps
           command: |
-            python3 -m pip install -U pip
-            python3 -m pip install -r requirements.txt
-            python3 -m pip install -r requirements-dev.txt
             python3 -m pip install tzdata
             python3 -m pip install -r examples/python/ml/requirements.txt
-      - run:
-          name: Setup GCS
-          command: |
-            echo ${gcs_content} > ../gcs.data
-            ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json --min_download
-      - run:
-          name: "build"
-          command: bazel build //examples/python/ml:ml_test -c opt --ui_event_filters=-info,-debug,-warning --jobs 20
-      - run:
-          name: "test"
-          command: |
-            set +e
-            declare -i test_status
-            bazel test --test_env LD_LIBRARY_PATH=/root/miniconda3/lib/ //examples/python/ml:ml_test -c opt --ui_event_filters=-info,-debug,-warning --test_timeout=3600 --test_output=errors| tee test_result.log; test_status=${PIPESTATUS[0]}
-
-            sh ../devtools/rename-junit-xml.sh
-            find bazel-testlogs/ -type f -name "test.log"  -print0 | xargs -0 tar -cvzf test_logs.tar.gz
-
-            exit ${test_status}
-      - store_test_results:
-          path: test-results
-      - store_artifacts:
-          path: test_logs.tar.gz
+      - build_and_test:
+          targets: //examples/python/ml:ml_test
+          extra_bazel_test_args: --test_env LD_LIBRARY_PATH=/root/miniconda3/lib/
   macOS_ut:
     macos:
-      xcode: 15.1
+      xcode: 15.1.0
     resource_class: macos.m1.large.gen1
     steps:
       - checkout
-      - run:
-          name: Checkout devtools
-          command: git clone https://github.com/secretflow/devtools.git ../devtools
       - run:
           name: "Install homebrew dependencies"
           command: brew install bazelisk cmake ninja libomp wget md5sha1sum
@@ -151,45 +127,21 @@ jobs:
             conda init bash zsh
             pip install -r requirements.txt
             pip install -r requirements-dev.txt
-      - run:
-          name: Setup GCS
-          command: |
-            echo ${gcs_content} > ../gcs.data
-            ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json --min_download
-      - run:
-          name: "build"
-          command: bazel build //... -c opt --ui_event_filters=-info,-debug,-warning
-      - run:
-          name: "test"
-          command: |
-            set +e
-            declare -i test_status
-            bazel test //... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors | tee test_result.log; test_status=${PIPESTATUS[0]}
-
-            sh ../devtools/rename-junit-xml.sh
-            find bazel-bin/ -perm +111 -type f -name "*_test"  -print0 | xargs -0 tar -cvzf test_binary.tar.gz
-            find bazel-testlogs/ -type f -name "test.log"  -print0 | xargs -0 tar -cvzf test_logs.tar.gz
-
-            exit ${test_status}
-      - store_test_results:
-          path: test-results
-      - store_artifacts:
-          path: test_binary.tar.gz
-      - store_artifacts:
-          path: test_logs.tar.gz
+      - build_and_test
 # Invoke jobs via workflows
 # See: https://circleci.com/docs/2.0/configuration-reference/#workflows
 workflows:
   unittest:
     when: << pipeline.parameters.build-and-run >>
     jobs:
       - linux_ut:
+          name: linux_ut-<<matrix.resource_class>>
           matrix:
             parameters:
-              resource_class: ["2xlarge", "arm.2xlarge"]
+              resource_class: ["2xlarge+", "arm.2xlarge"]
       - linux_ml_ut:
           requires:
-            - linux_ut
+            - linux_ut-2xlarge+
           filters:
             branches:
               only: