ci: check whether plugin/gem were installed during upgrading package

When major upgrade process was executed, manually installed plugin/gem should be installed in advance. This PR test that mechanism will be launched. Signed-off-by: Kentaro Hayashi <hayashi@clear-code.com>
fluent · Oct 22, 2024 · 5dfe5c2 · 5dfe5c2
1 parent 22f007f
commit 5dfe5c2
Show file tree

Hide file tree

Showing 15 changed files with 148 additions and 12 deletions.
diff --git a/.github/workflows/apt.yml b/.github/workflows/apt.yml
@@ -18,6 +18,7 @@ jobs:
           - Ubuntu Focal amd64
           - Ubuntu Jammy amd64
           - Ubuntu Noble amd64
+        upgrade_to: [5, 6]
         include:
           - label: Debian bullseye amd64
             rake-job: debian-bullseye
@@ -46,6 +47,13 @@ jobs:
       - name: Build deb with Docker
         run: |
           rake apt:build APT_TARGETS=${{ matrix.rake-job }}
+      - name: Build v6 deb with Docker
+        if: ${{ matrix.upgrade_to == 6 }}
+        run: |
+          git config user.email "fluentd@googlegroups.com"
+          git config user.name "Fluentd developers"
+          git am fluent-package/bump-version-v6.patch
+          rake apt:build APT_TARGETS=${{ matrix.rake-job }}
       - name: Upload fluent-package deb
         uses: actions/upload-artifact@master
         with:
@@ -63,9 +71,11 @@ jobs:
           path: fluent-lts-apt-source/apt/repositories
       # TODO move the following steps to "Test" job
       - name: Check Package Size
+        if: ${{ matrix.upgrade_to == 5 }}
         run: |
           fluent-package/apt/pkgsize-test.sh ${{ matrix.rake-job }} amd64
       - name: Installation Test
+        if: ${{ matrix.upgrade_to == 5 }}
         run: |
           mkdir -p .bundle
           docker run \
@@ -76,6 +86,7 @@ jobs:
           ${{ matrix.test-docker-image }} \
           /fluentd/fluent-package/apt/install-test.sh
       - name: Piuparts (Install/Remove/Upgrade) Test
+        if: ${{ matrix.upgrade_to == 5 }}
         run: |
           mkdir -p .bundle
           docker run \
@@ -87,6 +98,7 @@ jobs:
           ${{ matrix.test-docker-image }} \
           /fluentd/fluent-package/apt/piuparts-test.sh
       - name: Serverspec Test
+        if: ${{ matrix.upgrade_to == 5 }}
         run: |
           mkdir -p .bundle
           docker run \
@@ -97,6 +109,7 @@ jobs:
           ${{ matrix.test-docker-image }} \
           /fluentd/fluent-package/apt/serverspec-test.sh
       - name: Confluent Test
+        if: ${{ matrix.upgrade_to == 5 }}
         run: |
           mkdir -p .bundle
           docker run \
@@ -107,6 +120,7 @@ jobs:
           ${{ matrix.test-docker-image }} \
           /fluentd/fluent-package/apt/confluent-test.sh
       - name: Binstubs Test
+        if: ${{ matrix.upgrade_to == 5 }}
         run: |
           mkdir -p .bundle
           docker run \
@@ -138,6 +152,7 @@ jobs:
           - "install-newly.sh local"
           - "install-newly.sh v5"
           - "install-newly.sh lts"
+          - "install-missing-plugins.sh"
         include:
           - label: Debian bullseye amd64
             rake-job: debian-bullseye

diff --git a/.github/workflows/yum.yml b/.github/workflows/yum.yml
@@ -17,6 +17,7 @@ jobs:
           - AlmaLinux 9 x86_64
           - Amazon Linux 2 x86_64
           - Amazon Linux 2023 x86_64
+        upgrade_to: [5, 6]
         include:
           - label: RockyLinux 8 x86_64
             rake-job: rockylinux-8
@@ -46,16 +47,25 @@ jobs:
       - name: Build rpm with Docker
         run: |
           rake yum:build YUM_TARGETS=${{ matrix.rake-job }}
+      - name: Build v6 rpm with Docker
+        if: ${{ matrix.upgrade_to == 6 }}
+        run: |
+          git config user.email "fluentd@googlegroups.com"
+          git config user.name "Fluentd developers"
+          git am fluent-package/bump-version-v6.patch
+          rake yum:build YUM_TARGETS=${{ matrix.rake-job }}
       - name: Upload fluent-package rpm
         uses: actions/upload-artifact@v4
         with:
           name: packages-${{ matrix.rake-job }}
           path: fluent-package/yum/repositories
       # TODO move the following steps to "Test" job
       - name: Check Package Size
+        if: ${{ matrix.upgrade_to == 5 }}
         run: |
           fluent-package/yum/pkgsize-test.sh ${{ matrix.rake-job }} x86_64
       - name: Installation Test
+        if: ${{ matrix.upgrade_to == 5 }}
         run: |
           mkdir -p .bundle
           docker run \
@@ -67,6 +77,7 @@ jobs:
           ${{ matrix.test-docker-image }} \
           /fluentd/fluent-package/yum/install-test.sh
       - name: Serverspec Test
+        if: ${{ matrix.upgrade_to == 5 }}
         run: |
           mkdir -p .bundle
           docker run \
@@ -78,6 +89,7 @@ jobs:
           ${{ matrix.test-docker-image }} \
           /fluentd/fluent-package/yum/serverspec-test.sh
       - name: Confluent Test
+        if: ${{ matrix.upgrade_to == 5 }}
         run: |
           mkdir -p .bundle
           docker run \
@@ -88,6 +100,7 @@ jobs:
           ${{ matrix.test-docker-image }} \
           /fluentd/fluent-package/yum/confluent-test.sh
       - name: Binstubs Test
+        if: ${{ matrix.upgrade_to == 5 }}
         run: |
           mkdir -p .bundle
           docker run \
@@ -117,6 +130,7 @@ jobs:
           - "install-newly.sh local"
           - "install-newly.sh v5"
           - "install-newly.sh lts"
+          - "install-missing-plugins.sh"
         include:
           - label: AmazonLinux 2 x86_64
             rake-job: amazonlinux-2

diff --git a/fluent-package/apt/systemd-test/downgrade-to-v4.sh b/fluent-package/apt/systemd-test/downgrade-to-v4.sh
@@ -17,7 +17,7 @@ systemctl status --no-pager td-agent
 
 # Ensure to install the current
 sudo apt install -V -y \
-    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
+    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_5*_${architecture}.deb
 
 # td-agent.service is already masked (link to /dev/null), and remove td-agent.service alias not to conflict with v4
 sudo systemctl unmask td-agent

diff --git a/fluent-package/apt/systemd-test/install-missing-plugins.sh b/fluent-package/apt/systemd-test/install-missing-plugins.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -exu
+
+. $(dirname $0)/../commonvar.sh
+
+# Install the current
+sudo apt install -V -y \
+    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_5*_${architecture}.deb
+
+# Install plugin manually (plugin and gem)
+sudo /opt/fluent/bin/fluent-gem install --no-document fluent-plugin-concat
+sudo /opt/fluent/bin/fluent-gem install --no-document linux-utmpx
+
+# Install next major version
+sudo apt install -V -y \
+    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_6*_${architecture}.deb
+
+# Test: Check whether plugin/gem were installed during upgrading
+/opt/fluent/bin/fluent-gem list | grep fluent-plugin-concat
+/opt/fluent/bin/fluent-gem list | grep linux-utmpx
diff --git a/fluent-package/apt/systemd-test/install-newly.sh b/fluent-package/apt/systemd-test/install-newly.sh
@@ -7,7 +7,7 @@ set -exu
 case $1 in
   local)
     sudo apt install -V -y \
-      /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
+      /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_5*_${architecture}.deb
     ;;
   v5)
     curl --fail --silent --show-error --location https://toolbelt.treasuredata.com/sh/install-${distribution}-${code_name}-fluent-package5.sh | sh

diff --git a/fluent-package/apt/systemd-test/update-from-v4.sh b/fluent-package/apt/systemd-test/update-from-v4.sh
@@ -29,7 +29,7 @@ done
 
 # Install the current
 sudo apt install -V -y \
-    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
+    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_5*_${architecture}.deb
 
 # Test: service status
 systemctl status --no-pager fluentd

diff --git a/fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh b/fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh
@@ -29,7 +29,7 @@ done
 
 # Install the current
 sudo apt install -V -y \
-    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
+    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_5*_${architecture}.deb
 systemctl status --no-pager fluentd
 
 sudo systemctl stop fluentd
@@ -40,7 +40,7 @@ systemctl status --no-pager fluentd
 systemctl status --no-pager td-agent
 
 # Make a dummy pacakge for the next version
-dpkg-deb -R /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb tmp
+dpkg-deb -R /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_5*_${architecture}.deb tmp
 last_ver=$(cat tmp/DEBIAN/control | grep "Version: " | sed -E "s/Version: ([0-9.]+)-([0-9]+)/\2/g")
 sed -i -E "s/Version: ([0-9.]+)-([0-9]+)/Version: \1-$(($last_ver+1))/g" tmp/DEBIAN/control
 dpkg-deb --build tmp next_version.deb

diff --git a/fluent-package/apt/systemd-test/update-to-next-version.sh b/fluent-package/apt/systemd-test/update-to-next-version.sh
@@ -6,10 +6,10 @@ set -exu
 
 # Install the current
 sudo apt install -V -y \
-    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
+    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_5*_${architecture}.deb
 
 # Make a dummy pacakge for the next version
-dpkg-deb -R /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb tmp
+dpkg-deb -R /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_5*_${architecture}.deb tmp
 last_ver=$(cat tmp/DEBIAN/control | grep "Version: " | sed -E "s/Version: ([0-9.]+)-([0-9]+)/\2/g")
 sed -i -E "s/Version: ([0-9.]+)-([0-9]+)/Version: \1-$(($last_ver+1))/g" tmp/DEBIAN/control
 # Bump up x.y.w.z to check whether FLUENT_PACKAGE_VERSION was updated correctly later

diff --git a/fluent-package/bump-version-v6.patch b/fluent-package/bump-version-v6.patch
@@ -0,0 +1,65 @@
+From 887c36b60822995aa889fea6a181f00975cdfff7 Mon Sep 17 00:00:00 2001
+From: Kentaro Hayashi <hayashi@clear-code.com>
+Date: Thu, 17 Oct 2024 19:45:02 +0900
+Subject: [PATCH 1/2] bump version
+
+Signed-off-by: Kentaro Hayashi <hayashi@clear-code.com>
+---
+ fluent-package/config.rb                  | 6 +++---
+ fluent-package/debian/changelog           | 6 ++++++
+ fluent-package/yum/fluent-package.spec.in | 3 +++
+ 3 files changed, 12 insertions(+), 3 deletions(-)
+
+diff --git a/fluent-package/config.rb b/fluent-package/config.rb
+index a9e13bc..e61bca4 100644
+--- a/fluent-package/config.rb
++++ b/fluent-package/config.rb
+@@ -1,5 +1,5 @@
+ PACKAGE_NAME = "fluent-package"
+-PACKAGE_VERSION = "5.1.0"
++PACKAGE_VERSION = "6.0.0"
+
+ # Keep internal path (/opt/td-agent) for package name migration
+ SERVICE_NAME = "fluentd"
+@@ -23,8 +23,8 @@ OPENSSL_FOR_MACOS_SHA256SUM = "6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d
+ BUNDLER_VERSION= "2.3.27"
+
+ # https://www.ruby-lang.org/en/downloads/ (tar.gz)
+-BUNDLED_RUBY_VERSION = "3.2.5"
+-BUNDLED_RUBY_SOURCE_SHA256SUM = "ef0610b498f60fb5cfd77b51adb3c10f4ca8ed9a17cb87c61e5bea314ac34a16"
++BUNDLED_RUBY_VERSION = "3.3.5"
++BUNDLED_RUBY_SOURCE_SHA256SUM = "3781a3504222c2f26cb4b9eb9c1a12dbf4944d366ce24a9ff8cf99ecbce75196"
+
+ BUNDLED_RUBY_PATCHES = [
+   # An example entry:
+diff --git a/fluent-package/debian/changelog b/fluent-package/debian/changelog
+index f24ec3e..0089f56 100644
+--- a/fluent-package/debian/changelog
++++ b/fluent-package/debian/changelog
+@@ -1,3 +1,9 @@
++fluent-package (6.0.0-1) unstable; urgency=low
++
++  * New upstream release.
++
++ -- Kentaro Hayashi <kenhys@xdump.org>  Tue, 8 Oct 2024 07:47:29 -0000
++
+ fluent-package (5.1.0-1) unstable; urgency=low
+
+   * New upstream release.
+diff --git a/fluent-package/yum/fluent-package.spec.in b/fluent-package/yum/fluent-package.spec.in
+index 4833f8e..da9da8c 100644
+--- a/fluent-package/yum/fluent-package.spec.in
++++ b/fluent-package/yum/fluent-package.spec.in
+@@ -478,6 +478,9 @@ fi
+ # NOTE: %{_tmpfilesdir} is available since CentOS 7
+ %attr(0755,fluentd,fluentd) %dir /tmp/@PACKAGE_DIR@
+ %changelog
++* Mon Oct 7 2024 Kentaro Hayashi <kenhys@xdump.org> - 6.0.0-1
++- New upstream release. (Dummy)
++
+ * Mon Jul 29 2024 Kentaro Hayashi <kenhys@xdump.org> - 5.1.0-1
+ - New upstream release.
+
+-- 
+2.45.2
+
diff --git a/fluent-package/yum/systemd-test/downgrade-to-v4.sh b/fluent-package/yum/systemd-test/downgrade-to-v4.sh
@@ -23,7 +23,7 @@ sudo systemctl stop td-agent
 
 # Install the current
 sudo $DNF install -y \
-    /host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm
+    /host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-5[0-9]*.rpm
 
 sudo systemctl daemon-reload
 sudo systemctl enable --now fluentd

diff --git a/fluent-package/yum/systemd-test/install-missing-plugins.sh b/fluent-package/yum/systemd-test/install-missing-plugins.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -exu
+
+. $(dirname $0)/commonvar.sh
+
+# Install the current
+package="/host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-5[0-9]*.rpm"
+sudo $DNF install -y $package
+
+# Install plugin manually (plugin and gem)
+sudo /opt/fluent/bin/fluent-gem install --no-document fluent-plugin-concat
+sudo /opt/fluent/bin/fluent-gem install --no-document linux-utmpx
+
+# Install next major version
+package="/host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-6[0-9]*.rpm"
+sudo $DNF install -y $package
+
+# Test: Check whether plugin/gem were installed during upgrading
+/opt/fluent/bin/fluent-gem list | grep fluent-plugin-concat
+/opt/fluent/bin/fluent-gem list | grep linux-utmpx
diff --git a/fluent-package/yum/systemd-test/install-newly.sh b/fluent-package/yum/systemd-test/install-newly.sh
@@ -7,7 +7,7 @@ set -exu
 case $1 in
   local)
     sudo $DNF install -y \
-      /host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm
+      /host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-5[0-9]*.rpm
     ;;
   v5)
     case $DISTRIBUTION in

diff --git a/fluent-package/yum/systemd-test/update-from-v4.sh b/fluent-package/yum/systemd-test/update-from-v4.sh
@@ -44,7 +44,7 @@ done
 
 # Install the current
 sudo $DNF install -y \
-    /host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm
+    /host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-5[0-9]*.rpm
 
 # Test: take over enabled state
 systemctl is-enabled fluentd

diff --git a/fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh b/fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh
@@ -43,7 +43,7 @@ for d in $(seq 1 10); do
 done
 
 # Install the current
-package="/host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm"
+package="/host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-5[0-9]*.rpm"
 sudo $DNF install -y $package
 systemctl status --no-pager fluentd # Migration process starts the service automatically
 

diff --git a/fluent-package/yum/systemd-test/update-to-next-version.sh b/fluent-package/yum/systemd-test/update-to-next-version.sh
@@ -5,7 +5,7 @@ set -exu
 . $(dirname $0)/commonvar.sh
 
 # Install the current
-package="/host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm"
+package="/host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-5[0-9]*.rpm"
 sudo $DNF install -y $package
 sudo systemctl enable --now fluentd
 systemctl status --no-pager fluentd