Zero downtime update

.github/workflows/apt.yml

@@ -11,7 +11,7 @@ concurrency:
 jobs:
   build:
     name: Build
-    timeout-minutes: 60
+    timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
@@ -54,16 +54,34 @@ jobs:
             fluent-package/apt/repositories
             fluent-apt-source/apt/repositories
             fluent-lts-apt-source/apt/repositories
+            v6-test/fluent-package/apt/repositories
           key: ${{ runner.os }}-cache-${{ matrix.rake-job }}-${{ hashFiles('**/config.rb', '**/Rakefile', '**/Gemfile*', 'fluent-package/templates/**', 'fluent-package/debian/**', 'fluent-package/apt/**/Dockerfile') }}
       - name: Build deb with Docker
         if: ${{ ! steps.cache-deb.outputs.cache-hit }}
         run: |
           rake apt:build APT_TARGETS=${{ matrix.rake-job }}
+      - uses: actions/checkout@master
+        if: ${{ ! steps.cache-deb.outputs.cache-hit }}
+        with:
+          path: v6-test
+      - name: Build v6 deb with Docker
+        if: ${{ ! steps.cache-deb.outputs.cache-hit }}
+        run: |
+          cd v6-test
+          git config user.email "fluentd@googlegroups.com"
+          git config user.name "Fluentd developers"
+          git am fluent-package/bump-version-v6.patch
+          rake apt:build APT_TARGETS=${{ matrix.rake-job }}
       - name: Upload fluent-package deb
         uses: actions/upload-artifact@master
         with:
           name: packages-${{ matrix.rake-job }}
           path: fluent-package/apt/repositories
+      - name: Upload v6 fluent-package deb
+        uses: actions/upload-artifact@master
+        with:
+          name: v6-packages-${{ matrix.rake-job }}
+          path: v6-test/fluent-package/apt/repositories
       - name: Upload fluent-apt-source deb
         uses: actions/upload-artifact@master
         with:
@@ -147,7 +165,9 @@ jobs:
           - "update-from-v4.sh local"
           - "update-from-v4.sh v5"
           - "update-from-v4.sh lts"
+          - "update-from-v5-lts.sh"
           - "downgrade-to-v4.sh"
+          - "downgrade-to-v5-lts.sh"
           - "install-newly.sh local"
           - "install-newly.sh v5"
           - "install-newly.sh lts"
@@ -157,6 +177,15 @@ jobs:
           - "update-to-next-version-service-status.sh enabled inactive"
           - "update-to-next-version-service-status.sh disabled active"
           - "update-to-next-version-service-status.sh disabled inactive"
+          - "update-to-next-version-with-auto-and-manual.sh"
+          - "update-to-next-major-version.sh auto active"
+          - "update-to-next-major-version.sh auto inactive"
+          - "update-to-next-major-version.sh manual active"
+          - "update-to-next-major-version.sh manual inactive"
+          - "update-to-next-major-version.sh etc active"
+          - "update-to-next-major-version.sh etc inactive"
+          - "update-without-data-lost.sh v5 v6"
+          - "update-without-data-lost.sh v6 v5"
         include:
           - label: Debian bullseye amd64
             rake-job: debian-bullseye
@@ -199,9 +228,31 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: packages-${{ matrix.rake-job }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: v6-packages-${{ matrix.rake-job }}
+          path: v6-test
       - uses: actions/download-artifact@v4
         with:
           name: packages-apt-source-${{ matrix.rake-job }}
       - uses: canonical/setup-lxd@v0.1.2
+      - name: Run diagnostic
+        run: |
+          uname -a
+          echo "::group::snap info lxd"
+          snap info lxd
+          echo "::endgroup::"
+          echo "::group::snap services lxd"
+          snap services lxd
+          echo "::endgroup::"
+          echo "::group::snap logs lxd"
+          sudo snap logs lxd
+          echo "::endgroup::"
+          echo "::group::lxc remote list"
+          lxc remote list
+          echo "::endgroup::"
+          echo "::group::lxc list images:"
+          lxc image list images:
+          echo "::endgroup::"
       - name: Run Test  ${{ matrix.test }} on ${{ matrix.lxc-image }}
         run: fluent-package/apt/systemd-test/test.sh ${{ matrix.lxc-image }} ${{ matrix.test }}

.github/workflows/yum.yml

@@ -11,7 +11,7 @@ concurrency:
 jobs:
   build:
     name: Build
-    timeout-minutes: 60
+    timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
@@ -50,17 +50,36 @@ jobs:
         uses: actions/cache@v4
         id: cache-rpm
         with:
-          path: fluent-package/yum/repositories
+          path: |
+            fluent-package/yum/repositories
+            v6-test/fluent-package/yum/repositories
           key: ${{ runner.os }}-cache-${{ matrix.rake-job }}-${{ hashFiles('**/config.rb', '**/Rakefile', '**/Gemfile*', '**/*.spec.in', 'fluent-package/templates/**', 'fluent-package/yum/**/Dockerfile') }}
       - name: Build rpm with Docker
         if: ${{ ! steps.cache-rpm.outputs.cache-hit }}
         run: |
           rake yum:build YUM_TARGETS=${{ matrix.rake-job }}
+      - uses: actions/checkout@master
+        if: ${{ ! steps.cache-rpm.outputs.cache-hit }}
+        with:
+          path: v6-test
+      - name: Build v6 rpm with Docker
+        if: ${{ ! steps.cache-rpm.outputs.cache-hit }}
+        run: |
+          cd v6-test
+          git config user.email "fluentd@googlegroups.com"
+          git config user.name "Fluentd developers"
+          git am fluent-package/bump-version-v6.patch
+          rake yum:build YUM_TARGETS=${{ matrix.rake-job }}
       - name: Upload fluent-package rpm
         uses: actions/upload-artifact@v4
         with:
           name: packages-${{ matrix.rake-job }}
           path: fluent-package/yum/repositories
+      - name: Upload v6 fluent-package rpm
+        uses: actions/upload-artifact@v4
+        with:
+          name: v6-packages-${{ matrix.rake-job }}
+          path: v6-test/fluent-package/yum/repositories
       # TODO move the following steps to "Test" job
       - name: Check Package Size
         run: |
@@ -121,7 +140,9 @@ jobs:
           - AmazonLinux 2023 x86_64
         test:
           - "update-from-v4.sh"
+          - "update-from-v5-lts.sh"
           - "downgrade-to-v4.sh"
+          - "downgrade-to-v5-lts.sh"
           - "install-newly.sh local"
           - "install-newly.sh v5"
           - "install-newly.sh lts"
@@ -131,6 +152,15 @@ jobs:
           - "update-to-next-version-service-status.sh enabled inactive"
           - "update-to-next-version-service-status.sh disabled active"
           - "update-to-next-version-service-status.sh disabled inactive"
+          - "update-to-next-version-with-auto-and-manual.sh"
+          - "update-to-next-major-version.sh auto active"
+          - "update-to-next-major-version.sh auto inactive"
+          - "update-to-next-major-version.sh manual active"
+          - "update-to-next-major-version.sh manual inactive"
+          - "update-to-next-major-version.sh etc active"
+          - "update-to-next-major-version.sh etc inactive"
+          - "update-without-data-lost.sh v5 v6"
+          - "update-without-data-lost.sh v6 v5"
         include:
           - label: AmazonLinux 2 x86_64
             rake-job: amazonlinux-2
@@ -150,6 +180,10 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: packages-${{ matrix.rake-job }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: v6-packages-${{ matrix.rake-job }}
+          path: v6-test
       - uses: canonical/setup-lxd@v0.1.2
       - name: Run diagnostic
         run: |
@@ -185,7 +219,9 @@ jobs:
           - AlmaLinux 9 x86_64
         test:
           - "update-from-v4.sh"
+          - "update-from-v5-lts.sh"
           - "downgrade-to-v4.sh"
+          - "downgrade-to-v5-lts.sh"
           - "install-newly.sh local"
           - "install-newly.sh v5"
           - "install-newly.sh lts"
@@ -195,6 +231,15 @@ jobs:
           - "update-to-next-version-service-status.sh enabled inactive"
           - "update-to-next-version-service-status.sh disabled active"
           - "update-to-next-version-service-status.sh disabled inactive"
+          - "update-to-next-version-with-auto-and-manual.sh"
+          - "update-to-next-major-version.sh auto active"
+          - "update-to-next-major-version.sh auto inactive"
+          - "update-to-next-major-version.sh manual active"
+          - "update-to-next-major-version.sh manual inactive"
+          - "update-to-next-major-version.sh etc active"
+          - "update-to-next-major-version.sh etc inactive"
+          - "update-without-data-lost.sh v5 v6"
+          - "update-without-data-lost.sh v6 v5"
         include:
           - label: RockyLinux 8 x86_64
             rake-job: rockylinux-8
@@ -207,6 +252,10 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: packages-${{ matrix.rake-job }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: v6-packages-${{ matrix.rake-job }}
+          path: v6-test
       - uses: canonical/setup-lxd@v0.1.2
       - name: Run Test ${{ matrix.test }} on ${{ matrix.lxc-image }}
         run: fluent-package/yum/systemd-test/test.sh ${{ matrix.lxc-image }} ${{ matrix.test }}

fluent-package/Rakefile

@@ -494,7 +494,7 @@ class BuildTask
         remove_needless_files
       end
 
-      debian_pkg_scripts = ["preinst", "postinst", "postrm"]
+      debian_pkg_scripts = ["preinst", "postinst", "prerm", "postrm"]
       debian_pkg_scripts.each do |script|
         CLEAN.include(File.join("..", "debian", script))
       end

fluent-package/apt/commonvar.sh

@@ -3,6 +3,8 @@ architecture=$(dpkg --print-architecture)
 repositories_dir=/fluentd/fluent-package/apt/repositories
 java_jdk=openjdk-11-jre
 td_agent_version=4.5.2
+fluent_package_lts_version=5.0.5
+
 case ${code_name} in
   xenial)
     distribution=ubuntu

fluent-package/apt/systemd-test/downgrade-to-v5-lts.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -exu
+
+. $(dirname $0)/../commonvar.sh
+
+# Install v5 LTS to register the repository
+curl --fail --silent --show-error --location https://toolbelt.treasuredata.com/sh/install-${distribution}-${code_name}-fluent-package5-lts.sh | sh
+
+sudo apt purge -y fluent-package
+
+# Install the current
+sudo apt install -V -y \
+    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
+
+# Test: service status
+systemctl status --no-pager fluentd
+systemctl status --no-pager td-agent
+main_pid=$(eval $(systemctl show td-agent --property=MainPID) && echo $MainPID)
+
+# Downgrade to v5 LTS
+apt install -V -y fluent-package=${fluent_package_lts_version}-1 --allow-downgrades
+
+systemctl status --no-pager fluentd
+systemctl status --no-pager td-agent
+
+# Fluentd should be restarted.
+# NOTE: Unlike RPM, the restart behavior depends on TO-side. So, it restarts.
+test $main_pid -ne $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID)

fluent-package/apt/systemd-test/test.sh

@@ -15,7 +15,7 @@ dir="/host/fluent-package/apt/systemd-test"
 set -eux
 
 echo "::group::Run test: launch $image"
-lxc launch $image target
+lxc launch $image target --debug
 sleep 5
 echo "::endgroup::"
 echo "::group::Run test: configure $image"

fluent-package/apt/systemd-test/update-from-v5-lts.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+set -exu
+
+. $(dirname $0)/../commonvar.sh
+
+
+# If it update from v5 LTS without stopping Fluentd, Fluentd will not be restarted.
+# Install v5 LTS
+curl --fail --silent --show-error --location https://toolbelt.treasuredata.com/sh/install-${distribution}-${code_name}-fluent-package5-lts.sh | sh
+
+systemctl status --no-pager fluentd
+systemctl status --no-pager td-agent
+main_pid=$(eval $(systemctl show td-agent --property=MainPID) && echo $MainPID)
+
+# Install the current
+sudo apt install -V -y \
+    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
+
+# Test: service status
+systemctl status --no-pager fluentd
+systemctl status --no-pager td-agent
+
+# Fluentd should NOT be restarted.
+test $main_pid -eq $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID)
+
+apt purge -y fluent-package
+
+# If it update from v5 LTS with stopping Fluentd, Fluentd will be started when service is enabled.
+# Install v5 LTS
+curl --fail --silent --show-error --location https://toolbelt.treasuredata.com/sh/install-${distribution}-${code_name}-fluent-package5-lts.sh | sh
+
+systemctl status --no-pager fluentd
+systemctl status --no-pager td-agent
+main_pid=$(eval $(systemctl show td-agent --property=MainPID) && echo $MainPID)
+
+systemctl stop fluentd
+
+# Install the current
+sudo apt install -V -y \
+    /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb
+
+systemctl status --no-pager fluentd
+systemctl status --no-pager td-agent
+
+# Fluentd should be started if service was stopped before update.
+test $main_pid -ne $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID)
+
+# Test: environmental variables
+pid=$(systemctl show fluentd --property=MainPID --value)
+env_vars=$(sudo sed -e 's/\x0/\n/g' /proc/$pid/environ)
+test $(eval $env_vars && echo $HOME) = "/var/lib/fluent"
+test $(eval $env_vars && echo $LOGNAME) = "_fluentd"
+test $(eval $env_vars && echo $USER) = "_fluentd"
+test $(eval $env_vars && echo $FLUENT_CONF) = "/etc/fluent/fluentd.conf"
+test $(eval $env_vars && echo $FLUENT_PACKAGE_LOG_FILE) = "/var/log/fluent/fluentd.log"
+test $(eval $env_vars && echo $FLUENT_PLUGIN) = "/etc/fluent/plugin"
+test $(eval $env_vars && echo $FLUENT_SOCKET) = "/var/run/fluent/fluentd.sock"
+
+# Test: No error logs
+# (v5 default config outputs 'warn' log, so we should check only 'error' and 'fatal' logs)
+sleep 3
+test -e /var/log/fluent/fluentd.log
+(! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
+
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd)
+(! sudo /usr/sbin/fluentd -v)
+sudo /usr/sbin/fluentd --dry-run