RPM packaging: cache CI, separate RPM and Docker architecture, fix cr…

…eating Passenger source tarball
phusion · Oct 13, 2024 · 3e4b598 · 3e4b598
1 parent f6da9ee
commit 3e4b598
Show file tree

Hide file tree

Showing 4 changed files with 87 additions and 29 deletions.
diff --git a/.github/workflows/rpm.yml b/.github/workflows/rpm.yml
@@ -21,30 +21,84 @@ jobs:
         run: echo "distros=[$(awk -F= '/DEFAULT_DISTROS/{print $2}' packaging/rpm/internal/lib/distro_info.sh | sed -e 's/ /", "/g')]" >> "$GITHUB_OUTPUT"
 
   test:
-    name: "Test ${{ matrix.distro }} ${{ matrix.arch.name }} packages"
+    name: "Test ${{ matrix.distro }} ${{ matrix.arch.rpm_arch }} packages"
     needs: define-matrix
     strategy:
       fail-fast: false
       matrix:
         distro: ${{ fromJSON(needs.define-matrix.outputs.distros) }}
         arch:
-          - name: amd64
+          - rpm_arch: x86_64
+            docker_arch: amd64
             runner: ubuntu-24.04
-          - name: arm64
+          - rpm_arch: aarch64
+            docker_arch: arm64
             runner: passenger-ubuntu-24.04-arm64-4cpu
     runs-on: ${{ matrix.arch.runner }}
-    env:
-      WORKSPACE: ${{ github.workspace }}
-      ARCHITECTURE: ${{ matrix.arch.name }}
-      CACHE_DIR: ${{ github.workspace }}/cache/rpm-test/${{ matrix.distro }}-${{ matrix.arch.name }}
-      DISTRIBUTION: ${{ matrix.distro }}
     steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: docker/setup-buildx-action@v3
-    - run: ./dev/ci/tests/rpm/run
-    - uses: actions/upload-artifact@v4
-      with:
-        name: rpm-${{ matrix.distro }}-${{ matrix.arch.name }}
-        path: 'output/${{ matrix.distro }}/*'
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - run: mkdir work cache output
+        working-directory: packaging/rpm
+
+      - name: Determine test distribution name
+        id: determine_test_distro
+        run: |
+          source packaging/rpm/internal/lib/distro_info.sh
+          TEST_DISTRO_NAME=$(el_name_to_distro_name "$DISTRIBUTION")
+          echo "name=$TEST_DISTRO_NAME" >> "$GITHUB_OUTPUT"
+        env:
+          DISTRIBUTION: ${{ matrix.distro }}
+
+      - name: Fetch cache
+        run: ./dev/ci/fetch-cache-az-blob-storage
+        env:
+          AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }}
+          CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }}
+          BLOB_NAME: "binaries/rpm-cache-${{ matrix.distro }}-${{ matrix.arch.rpm_arch }}.tar.zstd"
+          CACHE_PATH: packaging/rpm/cache
+          SUDO: true
+
+      - name: Build
+        run: ./build -w work -c cache -o output -p "$WORKSPACE" -d "$DISTRIBUTION" -a "$RPM_ARCH" -A "$DOCKER_ARCH" -R rpm:all
+        env:
+          WORKSPACE: ${{ github.workspace }}
+          DISTRIBUTION: ${{ matrix.distro }}
+          RPM_ARCH: ${{ matrix.arch.rpm_arch }}
+          DOCKER_ARCH: ${{ matrix.arch.docker_arch }}
+        working-directory: packaging/rpm
+
+      - name: Test
+        run: >
+          ./test
+          -p "$WORKSPACE"
+          -d "output/$DISTRIBUTION"
+          -c cache
+          -x "$TEST_DISTRO_NAME"
+          -a "$RPM_ARCH"
+          -A "$DOCKER_ARCH"
+          -j
+        env:
+          WORKSPACE: ${{ github.workspace }}
+          DISTRIBUTION: ${{ matrix.distro }}
+          TEST_DISTRO_NAME: ${{ steps.determine_test_distro.outputs.name }}
+          RPM_ARCH: ${{ matrix.arch.rpm_arch }}
+          DOCKER_ARCH: ${{ matrix.arch.docker_arch }}
+        working-directory: packaging/rpm
+
+      - name: Update cache
+        run: ./dev/ci/update-cache-az-blob-storage
+        env:
+          AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }}
+          CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }}
+          BLOB_NAME: "binaries/rpm-cache-${{ matrix.distro }}-${{ matrix.arch.rpm_arch }}.tar.zstd"
+          CACHE_PATH: packaging/rpm/cache
+          SUDO: true
+        if: always()
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: rpm-${{ matrix.distro }}-${{ matrix.arch.rpm_arch }}
+          path: 'packaging/rpm/output/${{ matrix.distro }}/*'
diff --git a/dev/ci/tests/rpm/Jenkinsfile b/dev/ci/tests/rpm/Jenkinsfile
@@ -1,10 +1,11 @@
-def setupTest(enablerFlag, distribution, architecture, block) {
+def setupTest(enablerFlag, distribution, rpmArch, dockerArch, block) {
   if (enablerFlag) {
-    node("linux && ${architecture}") {
+    node("linux && ${rpmArch}") {
       withEnv([
-        "CACHE_DIR=${env.JENKINS_HOME}/cache/${env.JOB_NAME}/${distribution}-${architecture}",
+        "CACHE_DIR=${env.JENKINS_HOME}/cache/${env.JOB_NAME}/${distribution}-${rpmArch}",
         "DISTRIBUTION=${distribution}",
-        "ARCHITECTURE=${architecture}"
+        "RPM_ARCH=${rpmArch}",
+        "DOCKER_ARCH=${dockerArch}"
       ], block)
     }
   } else {
@@ -48,25 +49,25 @@ pipeline {
         script {
           parallel(
             'el8 x86_64': {
-              setupTest(params.EL8, 'el8', 'x86_64') {
+              setupTest(params.EL8, 'el8', 'x86_64', 'amd64') {
                 checkout scm
                 sh './dev/ci/tests/rpm/run'
               }
             },
             'el9 x86_64': {
-              setupTest(params.EL9, 'el9', 'x86_64') {
+              setupTest(params.EL9, 'el9', 'x86_64', 'amd64') {
                 checkout scm
                 sh './dev/ci/tests/rpm/run'
               }
             },
             'el8 aarch64': {
-              setupTest(params.EL8, 'el8', 'aarch64') {
+              setupTest(params.EL8, 'el8', 'aarch64', 'arm64') {
                 checkout scm
                 sh './dev/ci/tests/rpm/run'
               }
             },
             'el9 aarch64': {
-              setupTest(params.EL9, 'el9', 'aarch64') {
+              setupTest(params.EL9, 'el9', 'aarch64', 'arm64') {
                 checkout scm
                 sh './dev/ci/tests/rpm/run'
               }

diff --git a/dev/ci/tests/rpm/run b/dev/ci/tests/rpm/run
@@ -29,7 +29,8 @@ source "./internal/lib/distro_info.sh"
 
 require_envvar WORKSPACE "$WORKSPACE"
 require_envvar DISTRIBUTION "$DISTRIBUTION"
-require_envvar ARCHITECTURE "$ARCHITECTURE"
+require_envvar RPM_ARCH "$RPM_ARCH"
+require_envvar DOCKER_ARCH "$DOCKER_ARCH"
 
 PASSENGER_ROOT="${PASSENGER_ROOT:-$WORKSPACE}"
 CACHE_DIR="${CACHE_DIR:-$WORKSPACE/cache}"
@@ -51,14 +52,16 @@ run ./build \
 	-o "$WORKSPACE/output" \
 	-p "$PASSENGER_ROOT" \
 	-d "$DISTRIBUTION" \
-	-a "$ARCHITECTURE" \
+	-a "$RPM_ARCH" \
+	-A "$DOCKER_ARCH" \
 	-R \
 	rpm:all
 run ./test \
 	-p "$PASSENGER_ROOT" \
 	-d "$WORKSPACE/output/$DISTRIBUTION" \
 	-c "$CACHE_DIR" \
 	-x "$TEST_DISTRO_NAME" \
-	-a "$ARCHITECTURE" \
+	-a "$RPM_ARCH" \
+	-A "$DOCKER_ARCH" \
 	-j \
 	$EXTRA_TEST_PARAMS
diff --git a/packaging/rpm b/packaging/rpm