Merge branch 'oss_main' into boost_mmap_deprecation

Author: LinZhihao-723

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,9 @@
 <!--
-Set the PR title to a meaningful commit message in imperative form. E.g.:
-
-clp-s: Don't add implicit wildcards ('*') at the beginning and the end of a query (fixes #390).
+Set the PR title to a meaningful commit message that:
+- follows the Conventional Commits specification (https://www.conventionalcommits.org).
+- is in imperative form.
+Example:
+fix: Don't add implicit wildcards ('*') at the beginning and the end of a query (fixes #390).
 -->
 
 # Description

.github/actions/clp-core-build/action.yaml

@@ -8,6 +8,9 @@ inputs:
   use_published_image:
     description: "Whether to use the published container image"
     required: true
+  use_shared_libs:
+    description: "Whether to build the core binaries by linking against shared libraries"
+    required: true
   upload_binaries:
     description: "Whether to upload the core binaries"
     required: true
@@ -18,10 +21,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - shell: "bash"
-      working-directory: "./components/core"
-      run: "./tools/scripts/deps-download/download-all.sh"
-
     - if: "inputs.use_published_image == 'false'"
       uses: "actions/download-artifact@v4"
       with:
@@ -46,13 +45,21 @@ runs:
         fi
       shell: "bash"
 
+    - run: "./tools/scripts/deps-download/init.sh"
+      shell: "bash"
+
     - run: >-
         docker run
         --user $(id -u):$(id -g)
-        --volume "$GITHUB_WORKSPACE/components/core":/mnt/clp
+        --volume "$GITHUB_WORKSPACE":/mnt/clp
         --workdir /mnt/clp
         ${{steps.get_image_props.outputs.qualified_image_name}}
-        /mnt/clp/tools/scripts/utils/build-and-run-unit-tests.sh . build
+        bash -c "task deps:core &&
+        python3 /mnt/clp/components/core/tools/scripts/utils/build-and-run-unit-tests.py
+        ${{inputs.use_shared_libs == 'true' && '--use-shared-libs' || ''}}
+        --source-dir /mnt/clp/components/core
+        --build-dir /mnt/clp/components/core/build
+        --num-jobs $(getconf _NPROCESSORS_ONLN)"
       shell: "bash"
 
     - if: "inputs.upload_binaries == 'true'"

.github/workflows/clp-core-build-macos.yaml

@@ -8,19 +8,28 @@ on:
       - "components/core/CMakeLists.txt"
       - "components/core/src/**"
       - "components/core/tests/**"
-      - "components/core/tools/scripts/lib_install/macos-12/**"
+      - "components/core/tools/scripts/lib_install/macos/**"
       - "components/core/tools/scripts/deps-download/**"
-      - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+      - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
+      - "deps-tasks.yml"
+      - "Taskfile.yml"
+      - "tools/scripts/deps-download/**"
   push:
     paths:
       - ".github/workflows/clp-core-build-macos.yaml"
       - "components/core/cmake/**"
       - "components/core/CMakeLists.txt"
       - "components/core/src/**"
       - "components/core/tests/**"
-      - "components/core/tools/scripts/lib_install/macos-12/**"
+      - "components/core/tools/scripts/lib_install/macos/**"
       - "components/core/tools/scripts/deps-download/**"
-      - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+      - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
+      - "deps-tasks.yml"
+      - "Taskfile.yml"
+      - "tools/scripts/deps-download/**"
+  schedule:
+    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
+    - cron: "15 0 * * *"
   workflow_dispatch:
 
 concurrency:
@@ -30,7 +39,11 @@ concurrency:
 
 jobs:
   build-macos:
-    runs-on: "macos-12"
+    strategy:
+      matrix:
+        runner: ["macos-13", "macos-14"]
+        use_shared_libs: [true, false]
+    runs-on: "${{matrix.runner}}"
     steps:
       - uses: "actions/checkout@v4"
         with:
@@ -40,20 +53,28 @@ jobs:
       - name: "Remove preinstalled binaries which conflict with brew's installs"
         run: |
           rm -f /usr/local/bin/2to3*
+          rm -f /usr/local/bin/go*
           rm -f /usr/local/bin/idle3*
           rm -f /usr/local/bin/pydoc3*
           rm -f /usr/local/bin/python3*
 
       - name: "Install dependencies"
-        run: "./components/core/tools/scripts/lib_install/macos-12/install-all.sh"
+        run: "./components/core/tools/scripts/lib_install/macos/install-all.sh"
 
-      - name: "Download source dependencies"
+      - run: "./tools/scripts/deps-download/init.sh"
+        shell: "bash"
+
+      - run: "task deps:core"
         shell: "bash"
-        working-directory: "./components/core"
-        run: "./tools/scripts/deps-download/download-all.sh"
 
       - name: "Build CLP-core and run unit tests"
         shell: "bash"
         working-directory: "./components/core"
         # NOTE: We omit the Stopwatch tests since GH's macOS runner is too slow
-        run: "./tools/scripts/utils/build-and-run-unit-tests.sh . build ~[Stopwatch]"
+        run: >-
+          python3 ./tools/scripts/utils/build-and-run-unit-tests.py
+          ${{matrix.use_shared_libs == 'true' && '--use-shared-libs' || ''}}
+          --source-dir .
+          --build-dir build
+          --num-jobs $(getconf _NPROCESSORS_ONLN)
+          --test-spec "~[Stopwatch]"

.github/workflows/clp-core-build.yaml

@@ -8,15 +8,24 @@ on:
       - ".github/workflows/clp-core-build.yaml"
       - ".gitmodules"
       - "components/core/**"
-      - "!components/core/tools/scripts/lib_install/macos-12/**"
+      - "deps-tasks.yml"
+      - "Taskfile.yml"
+      - "tools/scripts/deps-download/**"
+      - "!components/core/tools/scripts/lib_install/macos/**"
   push:
     paths:
       - ".github/actions/clp-core-build/action.yaml"
       - ".github/actions/clp-core-build-containers/action.yaml"
       - ".github/workflows/clp-core-build.yaml"
       - ".gitmodules"
       - "components/core/**"
-      - "!components/core/tools/scripts/lib_install/macos-12/**"
+      - "deps-tasks.yml"
+      - "Taskfile.yml"
+      - "tools/scripts/deps-download/**"
+      - "!components/core/tools/scripts/lib_install/macos/**"
+  schedule:
+    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
+    - cron: "15 0 * * *"
   workflow_dispatch:
 
 env:
@@ -30,7 +39,7 @@ jobs:
   filter-relevant-changes:
     runs-on: "ubuntu-latest"
     outputs:
-      centos74_image_changed: "${{steps.filter.outputs.centos74_image}}"
+      centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}"
       ubuntu_focal_image_changed: "${{steps.filter.outputs.ubuntu_focal_image}}"
       ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}"
       clp_changed: "${{steps.filter.outputs.clp}}"
@@ -57,12 +66,12 @@ jobs:
           # image (since it would be different from the published image).
           base: "main"
           filters: |
-            centos74_image:
+            centos_stream_9_image:
               - ".github/actions/**"
               - ".github/workflows/clp-core-build.yaml"
               - "components/core/tools/scripts/lib_install/*.sh"
-              - "components/core/tools/docker-images/clp-env-base-centos7.4/**"
-              - "components/core/tools/scripts/lib_install/centos7.4/**"
+              - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**"
+              - "components/core/tools/scripts/lib_install/centos-stream-9/**"
             ubuntu_focal_image:
               - ".github/actions/**"
               - ".github/workflows/clp-core-build.yaml"
@@ -83,11 +92,13 @@ jobs:
               - "components/core/CMakeLists.txt"
               - "components/core/src/**"
               - "components/core/tests/**"
-              - "components/core/tools/scripts/deps-download/**"
-              - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+              - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
+              - "deps-tasks.yml"
+              - "Taskfile.yml"
+              - "tools/scripts/deps-download/**"
 
-  centos74-deps-image:
-    if: "needs.filter-relevant-changes.outputs.centos74_image_changed == 'true'"
+  centos-stream-9-deps-image:
+    if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'"
     needs: "filter-relevant-changes"
     runs-on: "ubuntu-latest"
     steps:
@@ -101,7 +112,7 @@ jobs:
 
       - uses: "./.github/actions/clp-core-build-containers"
         env:
-          OS_NAME: "centos7.4"
+          OS_NAME: "centos-stream-9"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
           docker_context: "components/core"
@@ -161,14 +172,19 @@ jobs:
             ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}
           token: "${{secrets.GITHUB_TOKEN}}"
 
-  centos74-binaries:
+  centos-stream-9-binaries:
     # Run if the ancestor jobs succeeded OR they were skipped and clp was changed.
     if: >-
       success()
       || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true')
     needs:
-      - "centos74-deps-image"
+      - "centos-stream-9-deps-image"
       - "filter-relevant-changes"
+    strategy:
+      matrix:
+        use_shared_libs: [true, false]
+    name: "centos-stream-9-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -181,11 +197,12 @@ jobs:
 
       - uses: "./.github/actions/clp-core-build"
         env:
-          OS_NAME: "centos7.4"
+          OS_NAME: "centos-stream-9"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_shared_libs: "${{matrix.use_shared_libs}}"
           use_published_image: >-
-            ${{needs.filter-relevant-changes.outputs.centos74_image_changed == 'false'
+            ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
           upload_binaries: "false"
 
@@ -197,6 +214,15 @@ jobs:
     needs:
       - "filter-relevant-changes"
       - "ubuntu-focal-deps-image"
+    strategy:
+      matrix:
+        include:
+          - use_shared_libs: true
+            upload_binaries: false
+          - use_shared_libs: false
+            upload_binaries: true
+    name: "ubuntu-focal-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -212,10 +238,11 @@ jobs:
           OS_NAME: "ubuntu-focal"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_shared_libs: "${{matrix.use_shared_libs}}"
           use_published_image: >-
             ${{needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
-          upload_binaries: "true"
+          upload_binaries: "${{matrix.upload_binaries}}"
           binaries_artifact_name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
 
   ubuntu-jammy-binaries:
@@ -226,6 +253,11 @@ jobs:
     needs:
       - "filter-relevant-changes"
       - "ubuntu-jammy-deps-image"
+    strategy:
+      matrix:
+        use_shared_libs: [true, false]
+    name: "ubuntu-jammy-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -241,6 +273,7 @@ jobs:
           OS_NAME: "ubuntu-jammy"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_shared_libs: "${{matrix.use_shared_libs}}"
           use_published_image: >-
             ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}

.github/workflows/clp-docs.yaml

@@ -0,0 +1,41 @@
+name: "clp-docs"
+
+on:
+  pull_request:
+  push:
+  schedule:
+    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
+    - cron: "15 0 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: "${{github.workflow}}-${{github.ref}}"
+  # Cancel in-progress jobs for efficiency
+  cancel-in-progress: true
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: ["macos-latest", "ubuntu-latest"]
+    runs-on: "${{matrix.os}}"
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          submodules: "recursive"
+
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.10"
+
+      - name: "Install task"
+        shell: "bash"
+        run: "npm install -g @go-task/cli"
+
+      - if: "matrix.os == 'macos-latest'"
+        name: "Install coreutils (for md5sum)"
+        run: "brew install coreutils"
+
+      - name: "Build docs"
+        shell: "bash"
+        run: "task docs:site"

.github/workflows/clp-execution-image-build.yaml

@@ -11,6 +11,9 @@ on:
       - ".github/actions/clp-execution-image-build/action.yaml"
       - ".github/workflows/clp-execution-image-build.yaml"
       - "tools/docker-images/**/*"
+  schedule:
+    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
+    - cron: "15 0 * * *"
   workflow_dispatch:
 
 concurrency:

.github/workflows/clp-lint.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
   push:
   schedule:
-    # Run at midnight UTC every day with 15 minutes delay added to avoid high load periods
+    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
     - cron: "15 0 * * *"
   workflow_dispatch:
 

.github/workflows/clp-pr-title-checks.yaml

@@ -0,0 +1,23 @@
+name: "clp-pr-title-checks"
+
+on:
+  pull_request_target:
+    types: ["edited", "opened", "reopened"]
+    branches: ["main"]
+
+concurrency:
+  group: "${{github.workflow}}-${{github.ref}}"
+
+  # Cancel in-progress jobs for efficiency
+  cancel-in-progress: true
+
+jobs:
+  conventional-commits:
+    permissions:
+      # For amannn/action-semantic-pull-request
+      pull-requests: "read"
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: "amannn/action-semantic-pull-request@v5"
+        env:
+          GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}"